Skip to content
TheChatArchiveThe Chat Archive
gpt-4o-2024-08-06· OpenAIAi2 WildChat-4.8M (AI2)

#machinemodel.py import aiohttp import numpy as np import pandas as pd from…

  1. User

    #machine_model.py import aiohttp import numpy as np import pandas as pd from sklearn.feature_selection import SelectKBest, mutual_info_regression from sklearn.metrics import mean_absolute_error from sklearn.model_selection import RandomizedSearchCV, train_test_split, TimeSeriesSplit from sklearn.preprocessing import StandardScaler from sklearn.ensemble import RandomForestRegressor, GradientBoostingRegressor from sklearn.svm import SVR from xgboost import XGBRegressor from lightgbm import LGBMRegressor import tensorflow as tf from keras import Sequential from tensorflow.python.keras.layers.recurrent import LSTM from tensorflow.python.keras.layers import Dense, Dropout, Attention, Conv1D from keras.src.callbacks import EarlyStopping from tensorflow.python.keras.optimizer_v1 import Adam from statsmodels.tsa.arima.model import ARIMA from prophet import Prophet import talib import asyncio from typing import Dict, List, Tuple import logging from .fetch_data import DataFetcher # Assume this is appropriately implemented to fetch data from .models import Signal # Assume this is your Django model for saving signals import json

    Logger configuration

    logging.basicConfig(level=logging.INFO) logger = logging.getLogger(name)

    class MachineTradingBot: def init(self, symbols: List[str]): self.models = { 'random_forest': RandomForestRegressor(random_state=42), 'svm': SVR(), 'xgboost': XGBRegressor(random_state=42), 'lightgbm': LGBMRegressor(random_state=42), 'gradient_boosting': GradientBoostingRegressor(random_state=42), 'lstm': None, 'hybrid_cnn_lstm': None, 'arima': None, 'prophet': Prophet(), } self.scaler = StandardScaler() self.best_params = {} self.feature_importances = {} self.model_weights = {model: 1 / len(self.models) for model in self.models} self.model_performance = {model: [] for model in self.models} self.timeframes = ['5m', '15m', '1h'] self.symbols = symbols self.session = None self.fetcher = DataFetcher(symbols=symbols, timeframes=self.timeframes)

    async def initialize(self):
        self.session = aiohttp.ClientSession()
        await self.fetcher.initialize_exchange()
    
    async def fetch_data(self, symbol: str, timeframe: str) -> pd.DataFrame:
        # Fetch historical data using DataFetcher
        await self.fetcher.fetch_historical_data()
        # Placeholder for fetching real-time updates
    
    def preprocess_data(self, data: pd.DataFrame) -> pd.DataFrame:
        data = self.add_technical_indicators(data)
        data = self.add_fourier_features(data)
        data = self.add_lagged_features(data)
        data = self.add_market_regime_features(data)
        data.dropna(inplace=True)
        return data
    
    def add_technical_indicators(self, data: pd.DataFrame) -> pd.DataFrame:
        for col in ['open', 'high', 'low', 'close', 'volume']:
            if col not in data.columns:
                logger.warning(f"Column {col} not found in data. Skipping technical indicators.")
                return data
    
        try:
            data['rsi'] = talib.RSI(data['close'])
            data['macd'], data['macd_signal'], _ = talib.MACD(data['close'])
            data['bbands_upper'], data['bbands_middle'], data['bbands_lower'] = talib.BBANDS(data['close'])
            data['adx'] = talib.ADX(data['high'], data['low'], data['close'])
            data['cci'] = talib.CCI(data['high'], data['low'], data['close'])
            data['mfi'] = talib.MFI(data['high'], data['low'], data['close'], data['volume'])
        except Exception as e:
            logger.error(f"Error calculating technical indicators: {str(e)}")
        return data
    
    def add_fourier_features(self, data: pd.DataFrame) -> pd.DataFrame:
        for period in [5, 10, 22]:
            for order in range(1, 3):
                data[f'fourier_sin_{period}_{order}'] = np.sin(2 * np.pi * order * data.index.dayofyear / period)
                data[f'fourier_cos_{period}_{order}'] = np.cos(2 * np.pi * order * data.index.dayofyear / period)
        return data
    
    def add_lagged_features(self, data: pd.DataFrame) -> pd.DataFrame:
        for lag in [1, 3, 5, 7, 14, 30]:
            data[f'close_lag_{lag}'] = data['close'].shift(lag)
            data[f'volume_lag_{lag}'] = data['volume'].shift(lag)
        return data
    
    def add_market_regime_features(self, data: pd.DataFrame) -> pd.DataFrame:
        returns = data['close'].pct_change()
        volatility = returns.rolling(window=22).std()
        data['market_regime'] = pd.cut(volatility, bins=3, labels=['low', 'medium', 'high'])
        data = pd.get_dummies(data, columns=['market_regime'])
        return data
    
    def prepare_features(self, data: pd.DataFrame) -> Tuple[pd.DataFrame, pd.Series]:
        features = data.drop(['open', 'high', 'low', 'close', 'volume'], axis=1)
        target = data['close'].shift(-1)
        selector = SelectKBest(mutual_info_regression, k=50)
        features = pd.DataFrame(selector.fit_transform(features, target), columns=features.columns[selector.get_support()])
        return features, target
    
    async def train_models(self, data: pd.DataFrame):
        features, target = self.prepare_features(data)
        X_train, X_test, y_train, y_test = train_test_split(features, target, test_size=0.2, shuffle=False)
        X_train_scaled = self.scaler.fit_transform(X_train)
        X_test_scaled = self.scaler.transform(X_test)
    
        tasks = []
        for model_name in ['random_forest', 'svm', 'xgboost', 'lightgbm', 'gradient_boosting']:
            tasks.append(self.tune_and_train(model_name, X_train_scaled, y_train, X_test_scaled, y_test))
    
        tasks.append(self.train_lstm(X_train_scaled, y_train, X_test_scaled, y_test))
        tasks.append(self.train_hybrid_cnn_lstm(X_train_scaled, y_train, X_test_scaled, y_test))
        tasks.append(self.train_arima(data['close']))
    
        prophet_data = pd.DataFrame({'ds': data.index, 'y': data['close']})
        tasks.append(self.train_prophet(prophet_data))
    
        await asyncio.gather(*tasks)
        self.calculate_feature_importances(features.columns)
    
    async def tune_and_train(self, model_name: str, X_train: np.ndarray, y_train: pd.Series, X_test: np.ndarray, y_test: pd.Series):
        logger.info(f"Tuning and training {model_name}...")
        param_distributions = self.get_param_distributions(model_name)
    
        tscv = TimeSeriesSplit(n_splits=5)
        search = RandomizedSearchCV(
            self.models[model_name],
            param_distributions=param_distributions,
            n_iter=50,
            cv=tscv,
            random_state=42,
            n_jobs=-1,
            scoring='neg_mean_absolute_error'
        )
        search.fit(X_train, y_train)
        self.models[model_name] = search.best_estimator_
        self.best_params[model_name] = search.best_params_
    
        y_pred = self.models[model_name].predict(X_test)
        mae = mean_absolute_error(y_test, y_pred)
        self.model_performance[model_name].append(mae)
    
        logger.info(f"Best parameters for {model_name}: {search.best_params_}")
        logger.info(f"MAE for {model_name}: {mae}")
    
    async def train_lstm(self, X_train: np.ndarray, y_train: pd.Series, X_test: np.ndarray, y_test: pd.Series):
        model = Sequential([
            LSTM(100, activation='relu', input_shape=(X_train.shape[1], 1), return_sequences=True),
            Attention(),
            Dropout(0.2),
            LSTM(50, activation='relu'),
            Dropout(0.2),
            Dense(1)
        ])
        model.compile(optimizer=Adam(learning_rate=0.001), loss='mse')
    
        X_train_reshaped = X_train.reshape((X_train.shape[0], X_train.shape[1], 1))
        X_test_reshaped = X_test.reshape((X_test.shape[0], X_test.shape[1], 1))
    
        early_stopping = EarlyStopping(patience=10, restore_best_weights=True)
    
        model.fit(
            X_train_reshaped, y_train,
            epochs=100,
            batch_size=32,
            validation_split=0.2,
            callbacks=[early_stopping],
            verbose=0
        )
        y_pred = model.predict(X_test_reshaped)
        mae = mean_absolute_error(y_test, y_pred)
        self.model_performance['lstm'].append(mae)
        logger.info(f"MAE for LSTM: {mae}")
    
        self.models['lstm'] = model
    
    async def train_hybrid_cnn_lstm(self, X_train: np.ndarray, y_train: pd.Series, X_test: np.ndarray, y_test: pd.Series):
        model = Sequential([
            Conv1D(filters=64, kernel_size=3, activation='relu', input_shape=(X_train.shape[1], 1)),
            Conv1D(filters=64, kernel_size=3, activation='relu'),
            LSTM(50, activation='relu', return_sequences=True),
            Attention(),
            LSTM(50, activation='relu'),
            Dense(1)
        ])
        model.compile(optimizer=Adam(learning_rate=0.001), loss='mse')
    
        X_train_reshaped = X_train.reshape((X_train.shape[0], X_train.shape[1], 1))
        X_test_reshaped = X_test.reshape((X_test.shape[0], X_test.shape[1], 1))
    
        early_stopping = EarlyStopping(patience=10, restore_best_weights=True)
    
        model.fit(
            X_train_reshaped, y_train,
            epochs=100,
            batch_size=32,
            validation_split=0.2,
            callbacks=[early_stopping],
            verbose=0
        )
    
        y_pred = model.predict(X_test_reshaped)
        mae = mean_absolute_error(y_test, y_pred)
        self.model_performance['hybrid_cnn_lstm'].append(mae)
        logger.info(f"MAE for Hybrid CNN-LSTM: {mae}")
    
        self.models['hybrid_cnn_lstm'] = model
    
    async def train_arima(self, time_series: pd.Series):
        model = ARIMA(time_series, order=(5, 1, 0))
        fitted = model.fit()
    
        # Evaluate performance
        y_pred = fitted.forecast(steps=len(time_series))
        mae = mean_absolute_error(time_series, y_pred)
        self.model_performance['arima'].append(mae)
        logger.info(f"MAE for ARIMA: {mae}")
    
        self.models['arima'] = fitted
    
    async def train_prophet(self, prophet_data: pd.DataFrame):
        model = Prophet()
        model.fit(prophet_data)
    
        future = model.make_future_dataframe(periods=len(prophet_data))
        forecast = model.predict(future)
    
        y_pred = forecast['yhat'][-len(prophet_data):]
        mae = mean_absolute_error(prophet_data['y'], y_pred)
        self.model_performance['prophet'].append(mae)
        logger.info(f"MAE for Prophet: {mae}")
    
        self.models['prophet'] = model
    
    async def generate_signals(self, latest_data: pd.DataFrame) -> Tuple[Dict[str, str], Dict[str, float]]:
        features, _ = self.prepare_features(latest_data)
        scaled_features = self.scaler.transform(features)
    
        signals = {}
        confidences = {}
    
        for model_name in ['random_forest', 'svm', 'xgboost', 'lightgbm', 'gradient_boosting']:
            prediction = self.models[model_name].predict(scaled_features)
            signal_strength = prediction[-1] - latest_data['close'].iloc[-1]
            category = "strong buy" if signal_strength > 0.005 else "weak buy" if signal_strength > 0 else "strong sell" if signal_strength < -0.005 else "weak sell"
            signals[model_name] = category
            confidences[model_name] = abs(signal_strength) / latest_data['close'].iloc[-1]
    
        # LSTM Model
        lstm_features = scaled_features.reshape((scaled_features.shape[0], scaled_features.shape[1], 1))
        lstm_prediction = self.models['lstm'].predict(lstm_features)
        signal_strength_lstm = lstm_prediction[-1][0] - latest_data['close'].iloc[-1]
        category_lstm = "strong buy" if signal_strength_lstm > 0.005 else "weak buy" if signal_strength_lstm > 0 else "strong sell" if signal_strength_lstm < -0.005 else "weak sell"
        signals['lstm'] = category_lstm
        confidences['lstm'] = abs(signal_strength_lstm) / latest_data['close'].iloc[-1]
    
        # Hybrid CNN-LSTM Model
        hybrid_prediction = self.models['hybrid_cnn_lstm'].predict(lstm_features)
        signal_strength_hybrid = hybrid_prediction[-1][0] - latest_data['close'].iloc[-1]
        category_hybrid = "strong buy" if signal_strength_hybrid > 0.005 else "weak buy" if signal_strength_hybrid > 0 else "strong sell" if signal_strength_hybrid < -0.005 else "weak sell"
        signals['hybrid_cnn_lstm'] = category_hybrid
        confidences['hybrid_cnn_lstm'] = abs(signal_strength_hybrid) / latest_data['close'].iloc[-1]
    
        # ARIMA Model
        arima_forecast = self.models['arima'].forecast(steps=1)
        signal_strength_arima = arima_forecast[0] - latest_data['close'].iloc[-1]
        category_arima = "strong buy" if signal_strength_arima > 0.005 else "weak buy" if signal_strength_arima > 0 else "strong sell" if signal_strength_arima < -0.005 else "weak sell"
        signals['arima'] = category_arima
        confidences['arima'] = abs(signal_strength_arima) / latest_data['close'].iloc[-1]
    
        # Prophet Model
        prophet_forecast = self.models['prophet'].predict(pd.DataFrame({'ds': [latest_data.index[-1] + pd.Timedelta(minutes=5)]}))
        signal_strength_prophet = prophet_forecast['yhat'].iloc[0] - latest_data['close'].iloc[-1]
        category_prophet = "strong buy" if signal_strength_prophet > 0.005 else "weak buy" if signal_strength_prophet > 0 else "strong sell" if signal_strength_prophet < -0.005 else "weak sell"
        signals['prophet'] = category_prophet
        confidences['prophet'] = abs(signal_strength_prophet) / latest_data['close'].iloc[-1]
    
        return signals, confidences
    
    async def concurrent_signal_generation(self, symbol: str):
        tasks = []
        for timeframe in self.timeframes:
            data = await self.fetch_data(symbol, timeframe)
            preprocessed_data = self.preprocess_data(data)
            await self.train_models(preprocessed_data)
            task = self.generate_signals(preprocessed_data.iloc[-30:])
            tasks.append(task)
    
        results = await asyncio.gather(*tasks)
        return results
    
    async def save_signals_to_db(self, symbol: str, timeframe: str, final_decision: str, signals: Dict[str, str], confidences: Dict[str, float]):
        Signal.objects.update_or_create(
            symbol=symbol,
            timeframe=timeframe,
            defaults={
                'decision': final_decision,
                'signals': json.dumps(signals),
                'confidences': json.dumps(confidences),
            }
        )
        logger.info(f"Signals saved for {symbol} {timeframe}")
    
    async def run(self):
        await self.initialize()
        for symbol in self.symbols:
            results = await self.concurrent_signal_generation(symbol)
            for timeframe, (signals, confidences) in zip(self.timeframes, results):
                final_decision = "buy" if sum(1 for s in signals.values() if 'buy' in s) > len(signals) / 2 else "sell"
                await self.save_signals_to_db(symbol, timeframe, final_decision, signals, confidences)
        await self.session.close()
    

    if name == "main": symbols = ['BTC/USDT', 'ETH/USDT'] # Example list of symbols bot = MachineTradingBot(symbols) asyncio.run(bot.run())

    Between the code above and the one below, wic one is likely to produce more accurate signals. What are the strengths and weaknesses of each and what would you recommend I use. Also, anything that I can do to improve it's accuracy further

    import asyncio import numpy as np import pandas as pd from datetime import datetime, timedelta from sklearn.ensemble import RandomForestRegressor, GradientBoostingRegressor from sklearn.preprocessing import StandardScaler, RobustScaler from sklearn.model_selection import TimeSeriesSplit from sklearn.metrics import mean_absolute_error, precision_score, recall_score, f1_score import lightgbm as lgb from xgboost import XGBRegressor import tensorflow as tf from tensorflow.keras.models import Sequential from tensorflow.keras.layers import LSTM, Dense, Dropout from tensorflow.keras.callbacks import EarlyStopping from tensorflow.keras.optimizers import Adam import logging from typing import Dict, List, Tuple, Optional from django.utils import timezone from django.db.models import Q from decimal import Decimal

    Assume these are your Django models

    from .models import PriceData, Signal, PerformanceMetrics

    logging.basicConfig(level=logging.INFO) logger = logging.getLogger(name)

    class EnhancedTradingBot: def init(self, symbols: List[str]): self.symbols = symbols self.timeframes = ['1m', '5m', '15m', '1h'] self.models = self._initialize_models() self.scalers = { 'price': RobustScaler(), 'volume': RobustScaler(), 'features': StandardScaler() } self.performance_metrics = {} self.volatility_windows = {'1m': 60, '5m': 24, '15m': 16, '1h': 24} self.min_success_rate = 65 # Minimum success rate to generate signals

    def _initialize_models(self) -> Dict:
        return {
            'random_forest': RandomForestRegressor(
                n_estimators=100,
                max_depth=10,
                min_samples_split=5,
                min_samples_leaf=2,
                n_jobs=-1,
                random_state=42
            ),
            'lightgbm': lgb.LGBMRegressor(
                n_estimators=100,
                learning_rate=0.05,
                max_depth=7,
                num_leaves=31,
                n_jobs=-1,
                random_state=42
            ),
            'xgboost': XGBRegressor(
                n_estimators=100,
                learning_rate=0.05,
                max_depth=7,
                n_jobs=-1,
                random_state=42
            ),
            'gradient_boosting': GradientBoostingRegressor(
                n_estimators=100,
                learning_rate=0.05,
                max_depth=5,
                random_state=42
            ),
            'lstm': None  # Will be initialized during training
        }
    
    async def fetch_historical_data(self, symbol: str, timeframe: str) -> pd.DataFrame:
        """Fetch historical data from Django ORM"""
        queryset = PriceData.objects.filter(
            symbol=symbol,
            timeframe=timeframe
        ).order_by('timestamp')
        
        df = pd.DataFrame.from_records(
            queryset.values('timestamp', 'open', 'high', 'low', 'close', 'volume')
        )
        df.set_index('timestamp', inplace=True)
        return df
    
    def calculate_dynamic_thresholds(self, data: pd.DataFrame, timeframe: str) -> Tuple[float, float]:
        """Calculate dynamic thresholds based on volatility"""
        returns = data['close'].pct_change()
        volatility = returns.rolling(window=self.volatility_windows[timeframe]).std()
        current_volatility = volatility.iloc[-1]
        
        # Adjust thresholds based on volatility
        base_threshold = 0.005
        volatility_multiplier = current_volatility / returns.std()
        
        buy_threshold = base_threshold * volatility_multiplier
        sell_threshold = -base_threshold * volatility_multiplier
        
        return buy_threshold, sell_threshold
    
    def create_features(self, data: pd.DataFrame) -> pd.DataFrame:
        """Create advanced features for ML models"""
        df = data.copy()
        
        # Price-based features
        df['returns'] = df['close'].pct_change()
        df['log_returns'] = np.log1p(df['returns'])
        
        # Volatility features
        df['volatility'] = df['returns'].rolling(window=20).std()
        df['volatility_change'] = df['volatility'].pct_change()
        
        # Volume-based features
        df['volume_ma'] = df['volume'].rolling(window=20).mean()
        df['volume_std'] = df['volume'].rolling(window=20).std()
        df['volume_ratio'] = df['volume'] / df['volume_ma']
        
        # Price momentum
        for window in [5, 10, 20, 30]:
            df[f'momentum_{window}'] = df['close'].pct_change(periods=window)
            df[f'volume_momentum_{window}'] = df['volume'].pct_change(periods=window)
        
        # Price differences
        df['price_range'] = df['high'] - df['low']
        df['price_range_ratio'] = df['price_range'] / df['close']
        
        # Target variable (future returns)
        df['target'] = df['close'].shift(-1) / df['close'] - 1
        
        return df.dropna()
    
    def prepare_lstm_data(self, data: pd.DataFrame, lookback: int = 60) -> Tuple[np.ndarray, np.ndarray]:
        """Prepare data for LSTM model"""
        features = data.drop(['target'], axis=1).values
        target = data['target'].values
        
        X, y = [], []
        for i in range(len(features) - lookback):
            X.append(features[i:(i + lookback)])
            y.append(target[i + lookback])
            
        return np.array(X), np.array(y)
    
    def create_lstm_model(self, input_shape: Tuple) -> Sequential:
        """Create LSTM model with optimized architecture"""
        model = Sequential([
            LSTM(50, input_shape=input_shape, return_sequences=True),
            Dropout(0.2),
            LSTM(30, return_sequences=False),
            Dropout(0.2),
            Dense(1)
        ])
        
        model.compile(
            optimizer=Adam(learning_rate=0.001),
            loss='mse',
            metrics=['mae']
        )
        return model
    
    async def train_models(self, data: pd.DataFrame, timeframe: str):
        """Train all models with the provided data"""
        features_df = self.create_features(data)
        
        # Split features and target
        X = features_df.drop(['target'], axis=1)
        y = features_df['target']
        
        # Scale features
        X_scaled = self.scalers['features'].fit_transform(X)
        
        # Time series split
        tscv = TimeSeriesSplit(n_splits=5)
        
        # Train traditional ML models
        for name, model in self.models.items():
            if name != 'lstm':
                logger.info(f"Training {name} for {timeframe}")
                model.fit(X_scaled, y)
                
        # Train LSTM
        X_lstm, y_lstm = self.prepare_lstm_data(features_df)
        self.models['lstm'] = self.create_lstm_model((X_lstm.shape[1], X_lstm.shape[2]))
        
        early_stopping = EarlyStopping(
            monitor='val_loss',
            patience=5,
            restore_best_weights=True
        )
        
        self.models['lstm'].fit(
            X_lstm, y_lstm,
            epochs=50,
            batch_size=32,
            validation_split=0.2,
            callbacks=[early_stopping],
            verbose=0
        )
    
    def calculate_support_resistance(self, data: pd.DataFrame) -> Tuple[float, float]:
        """Calculate support and resistance levels"""
        price_range = data['high'].max() - data['low'].min()
        support = data['low'].iloc[-1] - (price_range * 0.1)
        resistance = data['high'].iloc[-1] + (price_range * 0.1)
        return support, resistance
    
    def calculate_success_rate(self, predictions: Dict[str, float], confidence_scores: Dict[str, float]) -> float:
        """Calculate success rate based on model predictions and confidence"""
        weighted_confidence = sum(pred * conf for pred, conf in zip(predictions.values(), confidence_scores.values()))
        return min(abs(weighted_confidence) * 100, 95)  # Cap at 95%
    
    async def generate_signals(self, symbol: str, timeframe: str) -> Optional[Dict]:
        """Generate trading signals with comprehensive analysis"""
        try:
            # Fetch and prepare data
            data = await self.fetch_historical_data(symbol, timeframe)
            features_df = self.create_features(data)
            
            if len(features_df) < 100:  # Minimum required data
                return None
                
            X = features_df.drop(['target'], axis=1).iloc[-1:]
            X_scaled = self.scalers['features'].transform(X)
            
            # Get predictions from all models
            predictions = {}
            confidence_scores = {}
            
            for name, model in self.models.items():
                if name != 'lstm':
                    pred = model.predict(X_scaled)[0]
                    predictions[name] = pred
                    confidence_scores[name] = abs(pred)
                else:
                    X_lstm = self.prepare_lstm_data(features_df.iloc[-60:], lookback=60)[0]
                    pred = model.predict(X_lstm)[-1][0]
                    predictions[name] = pred
                    confidence_scores[name] = abs(pred)
            
            # Calculate dynamic thresholds
            buy_threshold, sell_threshold = self.calculate_dynamic_thresholds(data, timeframe)
            
            # Calculate weighted prediction
            weights = {
                'random_forest': 0.2,
                'lightgbm': 0.25,
                'xgboost': 0.25,
                'gradient_boosting': 0.15,
                'lstm': 0.15
            }
            
            weighted_prediction = sum(pred * weights[name] for name, pred in predictions.items())
            
            # Calculate success rate
            success_rate = self.calculate_success_rate(predictions, confidence_scores)
            
            if success_rate < self.min_success_rate:
                return None
                
            # Determine signal type
            signal_type = 'neutral'
            if weighted_prediction > buy_threshold:
                signal_type = 'strong_buy' if weighted_prediction > 2 * buy_threshold else 'buy'
            elif weighted_prediction < sell_threshold:
                signal_type = 'strong_sell' if weighted_prediction < 2 * sell_threshold else 'sell'
            
            # Calculate support and resistance
            support, resistance = self.calculate_support_resistance(data)
            
            # Prepare signal details
            current_price = float(data['close'].iloc[-1])
            
            signal_details = {
                'symbol': symbol,
                'timeframe': timeframe,
                'timestamp': timezone.now(),
                'price_timestamp': data.index[-1],
                'price': current_price,
                'change_24h': float(data['close'].pct_change(periods=24).iloc[-1] * 100),
                'signal': signal_type,
                'stop_loss': support if signal_type in ['buy', 'strong_buy'] else resistance,
                'take_profit': resistance if signal_type in ['buy', 'strong_buy'] else support,
                'success_rate': success_rate,
                'model_predictions': predictions,
                'confidence_scores': confidence_scores,
                'weighted_prediction': weighted_prediction,
                'thresholds': {
                    'buy': buy_threshold,
                    'sell': sell_threshold
                }
            }
            
            return signal_details
            
        except Exception as e:
            logger.error(f"Error generating signal for {symbol} {timeframe}: {str(e)}")
            return None
    
    async def save_signal(self, signal_details: Dict):
        """Save signal to database"""
        try:
            Signal.objects.create(
                symbol=signal_details['symbol'],
                timeframe=signal_details['timeframe'],
                timestamp=signal_details['timestamp'],
                price_timestamp=signal_details['price_timestamp'],
                price=Decimal(str(signal_details['price'])),
                change_24h=Decimal(str(signal_details['change_24h'])),
                signal_type=signal_details['signal'],
                stop_loss=Decimal(str(signal_details['stop_loss'])),
                take_profit=Decimal(str(signal_details['take_profit'])),
                success_rate=Decimal(str(signal_details['success_rate'])),
                details=signal_details
            )
        except Exception as e:
            logger.error(f"Error saving signal: {str(e)}")
    
    async def run(self):
        """Main execution loop"""
        try:
            tasks = []
            for symbol in self.symbols:
                for timeframe in self.timeframes:
                    # Train models
                    data = await self.fetch_historical_data(symbol, timeframe)
                    await self.train_models(data, timeframe)
                    
                    # Generate and save signals
                    tasks.append(self.generate_signals(symbol, timeframe))
            
            signals = await asyncio.gather(*tasks)
            
            # Save valid signals
            save_tasks = []
            for signal in signals:
                if signal:
                    save_tasks.append(self.save_signal(signal))
            
            if save_tasks:
                await asyncio.gather(*tasks)
                
        except Exception as e:
            logger.error(f"Error in main execution loop: {str(e)}")
    

    if name == "main": symbols = ['BTC/USDT', 'ETH/USDT'] # Example symbols bot = EnhancedTradingBot(symbols) asyncio.run(bot.run())

  2. Assistant

    Comparing the two trading bots, the "MachineTradingBot" and the "EnhancedTradingBot," here are some strengths, weaknesses, and potential recommendations:

    Strengths and Weaknesses

    MachineTradingBot:

    • Strengths:

      1. Diverse Model Collection: Utilizes a wide variety of models, including Prophet, ARIMA, and hybrid models like CNN-LSTM.
      2. Technical Indicators and Fourier Features: Incorporates technical indicators and Fourier features to offer comprehensive feature representation.
      3. Concurrency: Utilizes asyncio for concurrent operations, enhancing performance for real-time data processing.
    • Weaknesses:

      1. Complexity: High complexity due to many different models and features, which could lead to overfitting.
      2. Signal Categorization: Uses arbitrary thresholds that may not adapt well to changing market conditions.
      3. Scalability Issues: Due to the number of models, scalability can become an issue as more symbols or timeframes are added.

    EnhancedTradingBot:

    • Strengths:

      1. Dynamic Thresholds: Implements dynamic thresholds based on volatility, making it adaptive to different market conditions.
      2. Advanced Feature Generation: Focuses on sophisticated feature engineering including momentum and volatility features.
      3. Success Rate Filtering: Has a success rate threshold to filter out low-confidence predictions, potentially improving signal quality.
    • Weaknesses:

      1. Heavy Dependency on Static Weights: Uses fixed weights for model prediction aggregation which might not always capture the optimal combination.
      2. Limited Model Range: Fewer model types, potentially missing out on patterns that more diverse models could capture.
      3. Single Approach on Model Training: Relies on traditional time series split without consideration for more modern cross-validation techniques.

    Recommendations

    • Model Diversification: Consider combining elements of both approaches for a hybrid solution, utilizing model diversity from the MachineTradingBot with the adaptive mechanisms from the EnhancedTradingBot.

    • Adaptive Weighting: Implement an adaptive weighting scheme for models based on past performance to dynamically allocate weights.

    • Hyperparameter Optimization: Both models might benefit from a more systematic hyperparameter optimization using Bayesian optimization techniques rather than static or randomized search.

    • Real-Time Adaptation: Enrich real-time adaptability by integrating reinforcement learning strategies to continuously learn and adapt to the trading environment.

    • Backtesting and Validation: Rigorously backtest both models using historical data, evaluating their accuracy and robustness under different market conditions to choose the more suitable one.

    In conclusion, both models have their unique advantages and could be improved by integrating their best parts. If adapting just one, the EnhancedTradingBot could be more advantageous due to its dynamic features and adaptability, with further enhancements on model diversity and weighting schemes.

  3. User

    Between EnhancedTradingBot above and the one below. Wic is better import asyncio import numpy as np import pandas as pd from datetime import datetime, timedelta from sklearn.ensemble import RandomForestRegressor from sklearn.preprocessing import RobustScaler from sklearn.model_selection import TimeSeriesSplit from sklearn.metrics import mean_absolute_error import lightgbm as lgb from tensorflow.keras.models import Sequential from tensorflow.keras.layers import LSTM, Dense, Dropout from tensorflow.keras.callbacks import EarlyStopping from tensorflow.keras.optimizers import Adam import optuna import logging from typing import Dict, List, Optional from django.utils import timezone from django.db.models import Q from decimal import Decimal import psutil import gc

    Assume these are your Django models

    from .models import PriceData, Signal, PerformanceMetrics

    logging.basicConfig(level=logging.INFO) logger = logging.getLogger(name)

    class OptimizedTradingBot: def init(self, batch_size: int = 10): self.batch_size = batch_size # Process symbols in batches self.timeframes = ['15m', '1h'] # Reduced timeframes for better resource management self.models = {} self.scaler = RobustScaler() # Single scaler for simplicity self.performance_metrics = {} self.volatility_windows = {'15m': 16, '1h': 24} self.min_success_rate = 70 # Increased for better signal quality self.study_storage = {} self.best_params = {}

        # Resource management
        total_ram = psutil.virtual_memory().total
        self.memory_limit = total_ram * 0.6  # 60% of total RAM
        self.cpu_cores = min(psutil.cpu_count(logical=False), 4)  # Limited to 4 cores
        
        # Model selection (reduced to 3 most efficient models)
        self.model_types = ['random_forest', 'lightgbm', 'lstm']
    
    async def fetch_symbols(self) -> List[str]:
        """Fetch available trading pairs from database"""
        try:
            # Assuming PriceData model has a symbols field or method
            symbols = await PriceData.objects.values_list('symbol', flat=True).distinct()
            return list(symbols)
        except Exception as e:
            logger.error(f"Error fetching symbols: {str(e)}")
            return ['BTC/USDT', 'ETH/USDT']  # Fallback symbols
    
    async def fetch_historical_data(self, symbol: str, timeframe: str, 
                                  lookback_periods: int = 1000) -> pd.DataFrame:
        """Fetch historical price data with error handling and validation"""
        try:
            end_time = timezone.now()
            start_time = end_time - timedelta(
                hours=lookback_periods if timeframe == '1h' else lookback_periods//4
            )
            
            query = Q(symbol=symbol) & Q(timeframe=timeframe) & \
                   Q(timestamp__range=(start_time, end_time))
            
            data = await PriceData.objects.filter(query).order_by('timestamp').values()
            
            df = pd.DataFrame(data)
            
            if len(df) < lookback_periods * 0.9:  # Ensure sufficient data
                logger.warning(f"Insufficient data for {symbol} {timeframe}")
                return None
                
            return df
            
        except Exception as e:
            logger.error(f"Error fetching historical data: {str(e)}")
            return None
    
    def create_features(self, df: pd.DataFrame) -> pd.DataFrame:
        """Create enhanced technical features"""
        df = df.copy()
        
        # Price-based features
        df['returns'] = df['close'].pct_change()
        df['log_returns'] = np.log1p(df['returns'])
        
        # Volatility features
        df['volatility'] = df['returns'].rolling(window=20).std()
        df['volatility_ratio'] = df['volatility'] / df['volatility'].rolling(window=100).mean()
        
        # Volume features
        df['volume_ma'] = df['volume'].rolling(window=20).mean()
        df['volume_ratio'] = df['volume'] / df['volume_ma']
        
        # Price momentum
        df['momentum'] = df['close'].pct_change(periods=12)
        df['acceleration'] = df['momentum'].diff()
        
        # Trend features
        df['ma_20'] = df['close'].rolling(window=20).mean()
        df['ma_50'] = df['close'].rolling(window=50).mean()
        df['trend'] = (df['ma_20'] / df['ma_50']) - 1
        
        # Target variable (future returns)
        df['target'] = df['returns'].shift(-1)
        
        # Clean up
        df = df.dropna()
        
        return df
    
    def prepare_lstm_data(self, df: pd.DataFrame, sequence_length: int = 20) -> tuple:
        """Prepare data for LSTM model with memory efficiency"""
        features = df.drop(['target'], axis=1).values
        target = df['target'].values
        
        X, y = [], []
        for i in range(len(features) - sequence_length):
            X.append(features[i:i + sequence_length])
            y.append(target[i + sequence_length])
            
        return np.array(X), np.array(y)
    
    async def optimize_random_forest(self, trial: optuna.Trial, X: np.ndarray, y: np.ndarray) -> float:
        """Optimize RandomForest with reduced parameter space"""
        params = {
            'n_estimators': trial.suggest_int('rf_n_estimators', 50, 100),
            'max_depth': trial.suggest_int('rf_max_depth', 3, 10),
            'min_samples_split': trial.suggest_int('rf_min_samples_split', 2, 8)
        }
        
        model = RandomForestRegressor(**params, n_jobs=self.cpu_cores, random_state=42)
        return await self._evaluate_model(model, X, y)
    
    async def optimize_lightgbm(self, trial: optuna.Trial, X: np.ndarray, y: np.ndarray) -> float:
        """Optimize LightGBM with memory-efficient parameters"""
        params = {
            'n_estimators': trial.suggest_int('lgb_n_estimators', 50, 100),
            'max_depth': trial.suggest_int('lgb_max_depth', 3, 8),
            'learning_rate': trial.suggest_float('lgb_learning_rate', 1e-3, 0.1, log=True),
            'num_leaves': trial.suggest_int('lgb_num_leaves', 15, 31)
        }
        
        model = lgb.LGBMRegressor(**params, n_jobs=self.cpu_cores, random_state=42)
        return await self._evaluate_model(model, X, y)
    
    async def optimize_lstm(self, trial: optuna.Trial, X: np.ndarray, y: np.ndarray) -> float:
        """Optimize LSTM with reduced complexity"""
        params = {
            'units': trial.suggest_int('lstm_units', 32, 64),
            'dropout': trial.suggest_float('lstm_dropout', 0.1, 0.3),
            'learning_rate': trial.suggest_float('lstm_learning_rate', 1e-3, 1e-2, log=True),
            'batch_size': 32  # Fixed batch size for stability
        }
        
        model = Sequential([
            LSTM(params['units'], input_shape=(X.shape[1], X.shape[2])),
            Dropout(params['dropout']),
            Dense(1)
        ])
        
        model.compile(
            optimizer=Adam(learning_rate=params['learning_rate']),
            loss='mse'
        )
        
        early_stopping = EarlyStopping(monitor='val_loss', patience=3)
        
        tscv = TimeSeriesSplit(n_splits=3)
        scores = []
        
        for train_idx, val_idx in tscv.split(X):
            history = model.fit(
                X[train_idx], y[train_idx],
                epochs=20,  # Reduced epochs
                batch_size=params['batch_size'],
                validation_data=(X[val_idx], y[val_idx]),
                callbacks=[early_stopping],
                verbose=0
            )
            scores.append(min(history.history['val_loss']))
        
        return np.mean(scores)
    
    async def _evaluate_model(self, model, X: np.ndarray, y: np.ndarray) -> float:
        """Evaluate model performance with cross-validation"""
        tscv = TimeSeriesSplit(n_splits=3)
        scores = []
        
        for train_idx, val_idx in tscv.split(X):
            X_train, X_val = X[train_idx], X[val_idx]
            y_train, y_val = y[train_idx], y[val_idx]
            
            model.fit(X_train, y_train)
            pred = model.predict(X_val)
            score = mean_absolute_error(y_val, pred)
            scores.append(score)
        
        return np.mean(scores)
    
    async def optimize_models(self, data: pd.DataFrame, timeframe: str):
        """Run optimization with resource constraints"""
        features_df = self.create_features(data)
        X = features_df.drop(['target'], axis=1)
        y = features_df['target']
        
        X_scaled = self.scaler.fit_transform(X)
        X_lstm, y_lstm = self.prepare_lstm_data(features_df)
        
        n_trials = 30  # Reduced number of trials
        
        for model_type in self.model_types:
            if psutil.virtual_memory().available < self.memory_limit:
                logger.warning("Low memory. Cleaning up...")
                gc.collect()
            
            study = optuna.create_study(direction='minimize')
            
            if model_type == 'lstm':
                await self._run_optimization(study, model_type, X_lstm, y_lstm, n_trials)
            else:
                await self._run_optimization(study, model_type, X_scaled, y, n_trials)
            
            self.best_params[f"{timeframe}_{model_type}"] = study.best_params
            self.study_storage[f"{timeframe}_{model_type}"] = study
    
    async def _run_optimization(self, study, model_type: str, X: np.ndarray, y: np.ndarray, n_trials: int):
        """Run optimization for a single model"""
        optimize_func = getattr(self, f'optimize_{model_type}')
        study.optimize(
            lambda trial: asyncio.run(optimize_func(trial, X, y)),
            n_trials=n_trials
        )
    
    async def generate_signals(self, symbol: str, timeframe: str) -> Optional[Dict]:
        """Generate trading signals with enhanced validation"""
        try:
            data = await self.fetch_historical_data(symbol, timeframe)
            if data is None:
                return None
            
            features_df = self.create_features(data)
            X = features_df.drop(['target'], axis=1)
            X_scaled = self.scaler.transform(X)
            
            predictions = []
            weights = {'random_forest': 0.4, 'lightgbm': 0.4, 'lstm': 0.2}
            
            for model_type in self.model_types:
                if model_type == 'lstm':
                    X_lstm, _ = self.prepare_lstm_data(features_df)
                    pred = self.models[model_type].predict(X_lstm[-1:])
                else:
                    pred = self.models[model_type].predict(X_scaled[-1:])
                predictions.append(pred[0] * weights[model_type])
            
            ensemble_prediction = sum(predictions)
            
            # Generate signal with enhanced validation
            signal = self._validate_and_create_signal(
                symbol, timeframe, ensemble_prediction, data
            )
            
            return signal
            
        except Exception as e:
            logger.error(f"Error generating signals: {str(e)}")
            return None
    
    def _validate_and_create_signal(self, symbol: str, timeframe: str, 
                                  prediction: float, data: pd.DataFrame) -> Optional[Dict]:
        """Validate and create trading signal with risk management"""
        try:
            current_price = data['close'].iloc[-1]
            volatility = data['close'].pct_change().std() * np.sqrt(252)
            
            # Dynamic threshold based on volatility
            threshold = max(0.001, volatility * 0.1)
            
            if abs(prediction) < threshold:
                return None
            
            signal_type = 'buy' if prediction > 0 else 'sell'
            
            # Dynamic stop loss and take profit based on volatility
            stop_loss = current_price * (1 - volatility) if signal_type == 'buy' else current_price * (1 + volatility)
            take_profit = current_price * (1 + volatility * 1.5) if signal_type == 'buy' else current_price * (1 - volatility * 1.5)
            
            # Calculate success rate based on historical predictions
            success_rate = self._calculate_success_rate(data, prediction)
            
            if success_rate < self.min_success_rate:
                return None
            
            return {
                'symbol': symbol,
                'timeframe': timeframe,
                'timestamp': timezone.now(),
                'price_timestamp': data.index[-1],
                'price': current_price,
                'change_24h': data['close'].pct_change(24).iloc[-1],
                'signal': signal_type,
                'stop_loss': stop_loss,
                'take_profit': take_profit,
                'success_rate': success_rate,
                'volatility': volatility
            }
            
        except Exception as e:
            logger.error(f"Error validating signal: {str(e)}")
            return None
    

    def _calculate_success_rate(self, data: pd.DataFrame, current_prediction: float) -> float: """Calculate success rate of previous predictions""" try: predictions = [] actuals = []

            for i in range(len(data) - 20, len(data)):
                features = self.create_features(data.iloc[:i])
                X = features.drop(['target'], axis=1)
                X_scaled = self.scaler.transform(X)
                
                model_predictions = []
                for model_type in self.model_types:
                    if model_type == 'lstm':
                        X_lstm, _ = self.prepare_lstm_data(features)
                        pred = self.models[model_type].predict(X_lstm[-1:])
                    else:
                        pred = self.models[model_type].predict(X_scaled[-1:])
                    model_predictions.append(pred[0])
                
                avg_pred = np.mean(model_predictions)
                predictions.append(avg_pred > 0)
                actuals.append(data['returns'].iloc[i] > 0)
            
            return np.mean(np.array(predictions) == np.array(actuals)) * 100
            
        except Exception as e:
            logger.error(f"Error calculating success rate: {str(e)}")
            return 0.0
    
    async def save_signal(self, signal_details: Dict):
        """Save signal to database with enhanced error handling"""
        try:
            if not signal_details:
                return
                
            signal = Signal.objects.create(
                symbol=signal_details['symbol'],
                timeframe=signal_details['timeframe'],
                timestamp=signal_details['timestamp'],
                price_timestamp=signal_details['price_timestamp'],
                price=Decimal(str(signal_details['price'])),
                change_24h=Decimal(str(signal_details['change_24h'])),
                signal_type=signal_details['signal'],
                stop_loss=Decimal(str(signal_details['stop_loss'])),
                take_profit=Decimal(str(signal_details['take_profit'])),
                success_rate=Decimal(str(signal_details['success_rate'])),
                analysis_type='ml_ensemble',
                details={
                    'models_used': self.model_types,
                    'volatility': float(signal_details['volatility']),
                    'optimization_metrics': {
                        model: str(study.best_value)
                        for model, study in self.study_storage.items()
                        if model.startswith(signal_details['timeframe'])
                    }
                }
            )
            
            # Update performance metrics
            await self._update_performance_metrics(signal_details['symbol'], signal_details['timeframe'])
            
            return signal
            
        except Exception as e:
            logger.error(f"Error saving signal: {str(e)}")
            return None
    
    async def _update_performance_metrics(self, symbol: str, timeframe: str):
        """Update performance metrics for monitoring"""
        try:
            # Get recent signals
            recent_signals = Signal.objects.filter(
                symbol=symbol,
                timeframe=timeframe,
                timestamp__gte=timezone.now() - timedelta(days=7)
            )
            
            # Calculate metrics
            total_signals = len(recent_signals)
            successful_signals = len([s for s in recent_signals if s.is_successful()])
            
            if total_signals > 0:
                success_rate = (successful_signals / total_signals) * 100
            else:
                success_rate = 0
                
            # Update or create metrics
            PerformanceMetrics.objects.update_or_create(
                symbol=symbol,
                timeframe=timeframe,
                defaults={
                    'success_rate': Decimal(str(success_rate)),
                    'total_signals': total_signals,
                    'last_updated': timezone.now()
                }
            )
            
        except Exception as e:
            logger.error(f"Error updating performance metrics: {str(e)}")
    
    async def process_symbol_batch(self, symbols: List[str]):
        """Process a batch of symbols"""
        try:
            for timeframe in self.timeframes:
                for symbol in symbols:
                    if psutil.virtual_memory().available < self.memory_limit:
                        logger.warning("Low memory. Cleaning up...")
                        gc.collect()
                        await asyncio.sleep(1)  # Give system time to recover
                    
                    # Fetch and prepare data
                    data = await self.fetch_historical_data(symbol, timeframe)
                    if data is not None:
                        # Train models if needed
                        await self.optimize_models(data, timeframe)
                        
                        # Generate and save signals
                        signal = await self.generate_signals(symbol, timeframe)
                        if signal:
                            await self.save_signal(signal)
                    
                    await asyncio.sleep(0.1)  # Prevent overwhelming the system
                    
        except Exception as e:
            logger.error(f"Error processing symbol batch: {str(e)}")
    
    async def run(self):
        """Main execution loop with enhanced error handling and resource management"""
        try:
            # Fetch available symbols
            symbols = await self.fetch_symbols()
            if not symbols:
                logger.error("No symbols available for trading")
                return
                
            # Process symbols in batches
            symbol_batches = [symbols[i:i + self.batch_size] 
                            for i in range(0, len(symbols), self.batch_size)]
            
            for batch in symbol_batches:
                await self.process_symbol_batch(batch)
                
                # Memory management after each batch
                gc.collect()
                await asyncio.sleep(1)
                
            logger.info("Trading bot execution completed successfully")
            
        except Exception as e:
            logger.error(f"Error in main execution loop: {str(e)}")
        finally:
            # Cleanup
            gc.collect()
    
    async def start(self):
        """Start the trading bot with error handling"""
        try:
            logger.info("Starting trading bot...")
            while True:
                await self.run()
                
                # Wait before next iteration
                await asyncio.sleep(300)  # 5 minutes between runs
                
        except KeyboardInterrupt:
            logger.info("Trading bot stopped by user")
        except Exception as e:
            logger.error(f"Critical error in trading bot: {str(e)}")
        finally:
            logger.info("Trading bot shutdown complete")
    

    if name == "main": # Initialize and run the bot bot = OptimizedTradingBot(batch_size=5) # Process 5 symbols at a time asyncio.run(bot.start())

  4. Assistant

    When comparing the EnhancedTradingBot and the OptimizedTradingBot, it's essential to evaluate their strengths and weaknesses to determine which might be more suitable for producing accurate trading signals. Here are some considerations:

    Similarities

    • Data Handling: Both bots handle historical data fetching and feature engineering similarly.
    • Model Usage: Both utilize models like RandomForestRegressor, LightGBM, and an LSTM.
    • Signal Generation: Both generate signals by aggregating predictions from multiple models.

    Differences

    EnhancedTradingBot:

    • Strengths:

      1. Dynamic Thresholds: Implements dynamic thresholds based on volatility for signal determination, allowing the bot to adapt to market conditions.
      2. Rich Feature Set: Extensive use of sophisticated features such as momentum, volatility, and trends.
      3. Success Rate Filtering: Uses a success rate threshold for filtering signals, which can improve the quality of signals.
    • Weaknesses:

      1. Resource Utilization: May not be optimized for resource use and scalability.
      2. Static Model Weights: Uses fixed weights in model predictions aggregation, which could reduce adaptability.

    OptimizedTradingBot:

    • Strengths:

      1. Resource Management: Limits the use of system resources (CPU and RAM) and includes garbage collection to enhance performance efficiency.
      2. Batch Processing: Processes symbols in batches to optimize resource usage and system performance.
      3. Model Optimization: Employs optuna for hyperparameter tuning, potentially leading to more accurate models.
      4. Fewer Timeframes: Maintains focus on fewer timeframes to ensure better allocation of computational resources.
    • Weaknesses:

      1. Reduced Model Complexity: Limits feature complexity and the number of models, which might reduce identification of complex patterns.
      2. Simplified Training: Reduction in trials and epochs for efficiency might lead to underfitting.

    Recommendations

    Based on the features and design choices:

    • For Resource-Constrained Environments: The OptimizedTradingBot is likely a better choice due to its built-in memory and CPU management features. The use of optuna for efficient hyperparameter tuning can potentially improve model performance despite fewer trials and simpler architecture.
    • For Rich Feature Analysis and Market Adaptation: The EnhancedTradingBot would be more appropriate, especially with its dynamic thresholds and extensive feature engineering for adapting to market conditions.

    Improvement Suggestions

    1. Hybrid Approach: Consider incorporating the enhanced dynamic threshold and feature engineering techniques from EnhancedTradingBot into the OptimizedTradingBot for better market adaptability.
    2. Adaptive Weighting: Implement a more dynamic approach to model weighting based on real-time performance metrics or context-aware decisions.
    3. Comprehensive Backtesting: Conduct thorough backtesting under various market conditions using historical data to validate performance and improve predictive accuracy.

    Ultimately, if system resource constraints are a significant concern, then focusing on the OptimizedTradingBot with improvements could offer a balance between precision and performance efficiency.

  5. User

    I made other changes, how is it now? import asyncio import numpy as np import pandas as pd from datetime import datetime, timedelta from sklearn.ensemble import RandomForestRegressor from sklearn.preprocessing import RobustScaler from sklearn.model_selection import TimeSeriesSplit from sklearn.metrics import mean_absolute_error import lightgbm as lgb from keras import Sequential from tensorflow.python.keras.layers.recurrent import LSTM from tensorflow.python.keras.layers import Dense, Dropout, Attention, Conv1D from keras.src.callbacks import EarlyStopping from tensorflow.python.keras.optimizer_v1 import Adam from statsmodels.tsa.arima.model import ARIMA from xgboost import XGBRegressor import optuna import logging from typing import Dict, List, Optional from django.utils import timezone from django.db.models import Q from decimal import Decimal import psutil import gc

    Assume these are your Django models

    from .models import PriceData, Signal, PerformanceMetrics

    logging.basicConfig(level=logging.INFO) logger = logging.getLogger(name)

    class OptimizedTradingBot: def init(self, batch_size: int = 8): self.batch_size = batch_size self.timeframes = ['15m', '1h'] self.models = {} self.scaler = RobustScaler() self.performance_metrics = {} self.volatility_windows = {'15m': 16, '1h': 24} self.min_success_rate = 65 # Lowered for ensemble diversity self.study_storage = {} self.best_params = {}

        # Resource management for 8GB RAM
        total_ram = psutil.virtual_memory().total
        self.memory_limit = total_ram * 0.5  # 50% of total RAM
        self.cpu_cores = min(psutil.cpu_count(logical=False), 2)  # Limited to 2 cores for stability
        
        # Enhanced model selection (5 diverse models)
        self.model_types = [
            'random_forest',    # Traditional ML
            'xgboost',         # Traditional ML
            'lightgbm',        # Traditional ML
            'lstm',            # Deep Learning
            'arima'            # Time Series
        ]
        
        # Model weights for ensemble
        self.model_weights = {
            'random_forest': 0.25,
            'xgboost': 0.20,
            'lightgbm': 0.25,
            'lstm': 0.15,
            'arima': 0.15
        }
    
    async def fetch_symbols(self) -> List[str]:
        """Fetch available trading pairs from database"""
        try:
            # Assuming PriceData model has a symbols field or method
            symbols = await PriceData.objects.values_list('symbol', flat=True).distinct()
            return list(symbols)
        except Exception as e:
            logger.error(f"Error fetching symbols: {str(e)}")
            return ['BTC/USDT', 'ETH/USDT']  # Fallback symbols
    
    async def fetch_historical_data(self, symbol: str, timeframe: str, 
                                  lookback_periods: int = 1000) -> pd.DataFrame:
        """Fetch historical price data with error handling and validation"""
        try:
            end_time = timezone.now()
            start_time = end_time - timedelta(
                hours=lookback_periods if timeframe == '1h' else lookback_periods//4
            )
            
            query = Q(symbol=symbol) & Q(timeframe=timeframe) & \
                   Q(timestamp__range=(start_time, end_time))
            
            data = await PriceData.objects.filter(query).order_by('timestamp').values()
            
            df = pd.DataFrame(data)
            
            if len(df) < lookback_periods * 0.9:  # Ensure sufficient data
                logger.warning(f"Insufficient data for {symbol} {timeframe}")
                return None
                
            return df
            
        except Exception as e:
            logger.error(f"Error fetching historical data: {str(e)}")
            return None
    
    def create_features(self, df: pd.DataFrame) -> pd.DataFrame:
        """Create enhanced technical features"""
        df = df.copy()
        
        # Price-based features
        df['returns'] = df['close'].pct_change()
        df['log_returns'] = np.log1p(df['returns'])
        
        # Volatility features
        df['volatility'] = df['returns'].rolling(window=20).std()
        df['volatility_ratio'] = df['volatility'] / df['volatility'].rolling(window=100).mean()
        
        # Volume features
        df['volume_ma'] = df['volume'].rolling(window=20).mean()
        df['volume_ratio'] = df['volume'] / df['volume_ma']
        
        # Price momentum
        df['momentum'] = df['close'].pct_change(periods=12)
        df['acceleration'] = df['momentum'].diff()
        
        # Trend features
        df['ma_20'] = df['close'].rolling(window=20).mean()
        df['ma_50'] = df['close'].rolling(window=50).mean()
        df['trend'] = (df['ma_20'] / df['ma_50']) - 1
        
        # Target variable (future returns)
        df['target'] = df['returns'].shift(-1)
        
        # Clean up
        df = df.dropna()
        
        return df
    
    def prepare_lstm_data(self, df: pd.DataFrame, sequence_length: int = 20) -> tuple:
        """Prepare data for LSTM model with memory efficiency"""
        features = df.drop(['target'], axis=1).values
        target = df['target'].values
        
        X, y = [], []
        for i in range(len(features) - sequence_length):
            X.append(features[i:i + sequence_length])
            y.append(target[i + sequence_length])
            
        return np.array(X), np.array(y)
    
    async def optimize_random_forest(self, trial: optuna.Trial, X: np.ndarray, y: np.ndarray) -> float:
        """Optimize RandomForest with reduced parameter space"""
        params = {
            'n_estimators': trial.suggest_int('rf_n_estimators', 50, 100),
            'max_depth': trial.suggest_int('rf_max_depth', 3, 10),
            'min_samples_split': trial.suggest_int('rf_min_samples_split', 2, 8)
        }
        
        model = RandomForestRegressor(**params, n_jobs=self.cpu_cores, random_state=42)
        return await self._evaluate_model(model, X, y)
    
    async def optimize_lightgbm(self, trial: optuna.Trial, X: np.ndarray, y: np.ndarray) -> float:
        """Optimize LightGBM with memory-efficient parameters"""
        params = {
            'n_estimators': trial.suggest_int('lgb_n_estimators', 50, 100),
            'max_depth': trial.suggest_int('lgb_max_depth', 3, 8),
            'learning_rate': trial.suggest_float('lgb_learning_rate', 1e-3, 0.1, log=True),
            'num_leaves': trial.suggest_int('lgb_num_leaves', 15, 31)
        }
        
        model = lgb.LGBMRegressor(**params, n_jobs=self.cpu_cores, random_state=42)
        return await self._evaluate_model(model, X, y)
    
    async def optimize_lstm(self, trial: optuna.Trial, X: np.ndarray, y: np.ndarray) -> float:
        """Optimize LSTM with reduced complexity"""
        params = {
            'units': trial.suggest_int('lstm_units', 32, 64),
            'dropout': trial.suggest_float('lstm_dropout', 0.1, 0.3),
            'learning_rate': trial.suggest_float('lstm_learning_rate', 1e-3, 1e-2, log=True),
            'batch_size': 32  # Fixed batch size for stability
        }
        
        model = Sequential([
            LSTM(params['units'], input_shape=(X.shape[1], X.shape[2])),
            Dropout(params['dropout']),
            Dense(1)
        ])
        
        model.compile(
            optimizer=Adam(learning_rate=params['learning_rate']),
            loss='mse'
        )
        
        early_stopping = EarlyStopping(monitor='val_loss', patience=3)
        
        tscv = TimeSeriesSplit(n_splits=3)
        scores = []
        
        for train_idx, val_idx in tscv.split(X):
            history = model.fit(
                X[train_idx], y[train_idx],
                epochs=20,  # Reduced epochs
                batch_size=params['batch_size'],
                validation_data=(X[val_idx], y[val_idx]),
                callbacks=[early_stopping],
                verbose=0
            )
            scores.append(min(history.history['val_loss']))
        
        return np.mean(scores)
    
    async def optimize_xgboost(self, trial: optuna.Trial, X: np.ndarray, y: np.ndarray) -> float:
        """Optimize XGBoost with memory-efficient parameters"""
        params = {
            'n_estimators': trial.suggest_int('xgb_n_estimators', 50, 100),
            'max_depth': trial.suggest_int('xgb_max_depth', 3, 8),
            'learning_rate': trial.suggest_float('xgb_learning_rate', 1e-3, 0.1, log=True),
            'subsample': trial.suggest_float('xgb_subsample', 0.6, 0.9),
            'colsample_bytree': trial.suggest_float('xgb_colsample_bytree', 0.6, 0.9)
        }
        
        model = XGBRegressor(
            **params,
            n_jobs=self.cpu_cores,
            random_state=42,
            tree_method='hist',  # Memory efficient histogram-based algorithm
            gpu_id=None  # Disable GPU
        )
        return await self._evaluate_model(model, X, y)
    
    async def optimize_arima(self, trial: optuna.Trial, X: np.ndarray, y: np.ndarray) -> float:
        """Optimize ARIMA parameters"""
        params = {
            'p': trial.suggest_int('arima_p', 1, 3),
            'd': trial.suggest_int('arima_d', 0, 1),
            'q': trial.suggest_int('arima_q', 1, 3)
        }
        
        # Use only the last 500 observations for ARIMA to save memory
        y_subset = y[-500:]
        
        try:
            model = ARIMA(y_subset, order=(params['p'], params['d'], params['q']))
            fitted = model.fit()
            pred = fitted.forecast(steps=1)
            score = mean_absolute_error([y_subset[-1]], [pred[0]])
            return score
        except:
            return float('inf')
    
    async def optimize_models(self, data: pd.DataFrame, timeframe: str):
        """Run optimization with enhanced resource management"""
        features_df = self.create_features(data)
        X = features_df.drop(['target'], axis=1)
        y = features_df['target']
        
        X_scaled = self.scaler.fit_transform(X)
        X_lstm, y_lstm = self.prepare_lstm_data(features_df)
        
        n_trials = 20  # Further reduced trials for memory efficiency
        
        for model_type in self.model_types:
            # Memory cleanup before each model optimization
            if psutil.virtual_memory().available < self.memory_limit:
                logger.warning("Low memory. Cleaning up...")
                gc.collect()
                await asyncio.sleep(2)  # Give more time for memory cleanup
            
            study = optuna.create_study(direction='minimize')
            
            if model_type == 'lstm':
                await self._run_optimization(study, model_type, X_lstm, y_lstm, n_trials)
            elif model_type == 'arima':
                await self._run_optimization(study, model_type, None, y, n_trials)
            else:
                await self._run_optimization(study, model_type, X_scaled, y, n_trials)
            
            self.best_params[f"{timeframe}_{model_type}"] = study.best_params
            self.study_storage[f"{timeframe}_{model_type}"] = study
    
    async def _evaluate_model(self, model, X: np.ndarray, y: np.ndarray) -> float:
        """Evaluate model performance with cross-validation"""
        tscv = TimeSeriesSplit(n_splits=5)
        scores = []
        
        for train_idx, val_idx in tscv.split(X):
            X_train, X_val = X[train_idx], X[val_idx]
            y_train, y_val = y[train_idx], y[val_idx]
            
            model.fit(X_train, y_train)
            pred = model.predict(X_val)
            score = mean_absolute_error(y_val, pred)
            scores.append(score)
        
        return np.mean(scores)
    
    async def _run_optimization(self, study, model_type: str, X: np.ndarray, y: np.ndarray, n_trials: int):
        """Run optimization for a single model"""
        optimize_func = getattr(self, f'optimize_{model_type}')
        study.optimize(
            lambda trial: asyncio.run(optimize_func(trial, X, y)),
            n_trials=n_trials
        )
    
    async def generate_signals(self, symbol: str, timeframe: str) -> Optional[Dict]:
        """Generate trading signals using all 5 models in the ensemble"""
        try:
            data = await self.fetch_historical_data(symbol, timeframe)
            if data is None:
                return None
            
            features_df = self.create_features(data)
            X = features_df.drop(['target'], axis=1)
            X_scaled = self.scaler.transform(X)
            
            predictions = []
            weights = []
            
            # Process each model type
            for model_type in self.model_types:
                try:
                    if psutil.virtual_memory().available < self.memory_limit:
                        logger.warning("Low memory during prediction. Cleaning up...")
                        gc.collect()
                        await asyncio.sleep(0.5)
                    
                    if model_type == 'lstm':
                        # LSTM requires sequence data
                        X_lstm, _ = self.prepare_lstm_data(features_df)
                        pred = self.models[model_type].predict(X_lstm[-1:])
                        pred_value = pred[0]
                    
                    elif model_type == 'arima':
                        # ARIMA works directly with the close prices
                        # Use only recent data for memory efficiency
                        recent_close = data['close'].iloc[-500:]  # Last 500 points
                        model = ARIMA(
                            recent_close,
                            order=(
                                self.best_params[f"{timeframe}_arima"]['arima_p'],
                                self.best_params[f"{timeframe}_arima"]['arima_d'],
                                self.best_params[f"{timeframe}_arima"]['arima_q']
                            )
                        )
                        fitted = model.fit()
                        pred = fitted.forecast(steps=1)
                        # Convert ARIMA prediction to return format
                        current_price = recent_close.iloc[-1]
                        pred_value = (pred[0] - current_price) / current_price
                    
                    elif model_type == 'random_forest':
                        # Random Forest prediction
                        pred = self.models[model_type].predict(X_scaled[-1:])
                        pred_value = pred[0]
                    
                    elif model_type == 'xgboost':
                        # XGBoost prediction
                        pred = self.models[model_type].predict(X_scaled[-1:])
                        pred_value = pred[0]
                    
                    elif model_type == 'lightgbm':
                        # LightGBM prediction
                        pred = self.models[model_type].predict(X_scaled[-1:])
                        pred_value = pred[0]
                    
                    # Add prediction and weight if valid
                    if not np.isnan(pred_value) and not np.isinf(pred_value):
                        predictions.append(pred_value * self.model_weights[model_type])
                        weights.append(self.model_weights[model_type])
                    
                except Exception as e:
                    logger.error(f"Error in model {model_type}: {str(e)}")
                    continue
            
            # Check if we have enough valid predictions
            if len(predictions) < 3:  # Require at least 3 valid models
                logger.warning("Insufficient valid predictions")
                return None
            
            # Normalize weights for available predictions
            total_weight = sum(weights)
            if total_weight > 0:
                predictions = [p * (1/total_weight) for p in predictions]
            
            # Calculate ensemble prediction
            ensemble_prediction = sum(predictions)
            
            # Enhanced signal validation
            signal = await self._validate_and_create_signal(
                symbol, timeframe, ensemble_prediction, data
            )
            
            # Add model contribution details to signal
            if signal:
                signal['model_contributions'] = {
                    model_type: weight
                    for model_type, weight in zip(self.model_types, weights)
                    if weight > 0
                }
            
            return signal
            
        except Exception as e:
            logger.error(f"Error generating signals: {str(e)}")
            return None
    
    async def save_signal(self, signal_details: Dict):
        """Save signal to database with enhanced model contribution tracking"""
        try:
            if not signal_details:
                return
                
            # Extract model contributions before removing from signal details
            model_contributions = signal_details.pop('model_contributions', {})
                
            signal = Signal.objects.create(
                symbol=signal_details['symbol'],
                timeframe=signal_details['timeframe'],
                timestamp=signal_details['timestamp'],
                price_timestamp=signal_details['price_timestamp'],
                price=Decimal(str(signal_details['price'])),
                change_24h=Decimal(str(signal_details['change_24h'])),
                signal_type=signal_details['signal'],
                stop_loss=Decimal(str(signal_details['stop_loss'])),
                take_profit=Decimal(str(signal_details['take_profit'])),
                success_rate=Decimal(str(signal_details['success_rate'])),
                analysis_type='ml_ensemble',
                details={
                    'models_used': self.model_types,
                    'model_contributions': model_contributions,
                    'volatility': float(signal_details['volatility']),
                    'volume_score': float(signal_details['volume_score']),
                    'trend_score': float(signal_details['trend_score']),
                    'atr': float(signal_details['atr']),
                    'optimization_metrics': {
                        model: str(study.best_value)
                        for model, study in self.study_storage.items()
                        if model.startswith(signal_details['timeframe'])
                    }
                }
            )
            
            # Update performance metrics
            await self._update_performance_metrics(signal_details['symbol'], signal_details['timeframe'])
            
            return signal
            
        except Exception as e:
            logger.error(f"Error saving signal: {str(e)}")
            return None
    
    async def _validate_and_create_signal(self, symbol: str, timeframe: str, prediction: float, data: pd.DataFrame) -> Optional[Dict]:
        """Enhanced signal validation with additional risk metrics"""
        try:
            current_price = data['close'].iloc[-1]
            
            # Enhanced volatility calculation using different timeframes
            short_vol = data['close'].pct_change().rolling(window=12).std() * np.sqrt(252)
            long_vol = data['close'].pct_change().rolling(window=26).std() * np.sqrt(252)
            vol_ratio = short_vol.iloc[-1] / long_vol.iloc[-1]
            
            # Dynamic threshold based on volatility ratio
            threshold = max(0.001, short_vol.iloc[-1] * 0.15 * vol_ratio)
            
            if abs(prediction) < threshold:
                return None
            
            signal_type = 'buy' if prediction > 0 else 'sell'
            
            # Enhanced stop loss and take profit calculations
            atr = self._calculate_atr(data)
            stop_loss = current_price * (1 - atr * 2) if signal_type == 'buy' else current_price * (1 + atr * 2)
            take_profit = current_price * (1 + atr * 3) if signal_type == 'buy' else current_price * (1 - atr * 3)
            
            # Calculate confidence score
            success_rate = self._calculate_success_rate(data, prediction)
            volume_score = self._calculate_volume_score(data)
            trend_score = self._calculate_trend_score(data)
            
            confidence_score = (success_rate * 0.4 + volume_score * 0.3 + trend_score * 0.3)
            
            if confidence_score < self.min_success_rate:
                return None
            
            return {
                'symbol': symbol,
                'timeframe': timeframe,
                'timestamp': timezone.now(),
                'price_timestamp': data.index[-1],
                'price': current_price,
                'change_24h': data['close'].pct_change(24).iloc[-1],
                'signal': signal_type,
                'stop_loss': stop_loss,
                'take_profit': take_profit,
                'success_rate': confidence_score,
                'volatility': short_vol.iloc[-1],
                'volume_score': volume_score,
                'trend_score': trend_score,
                'atr': atr
            }
            
        except Exception as e:
            logger.error(f"Error validating signal: {str(e)}")
            return None
    
    def _calculate_atr(self, data: pd.DataFrame, period: int = 14) -> float:
        """Calculate Average True Range"""
        try:
            high = data['high']
            low = data['low']
            close = data['close']
            
            tr1 = high - low
            tr2 = abs(high - close.shift())
            tr3 = abs(low - close.shift())
            
            tr = pd.concat([tr1, tr2, tr3], axis=1).max(axis=1)
            atr = tr.rolling(window=period).mean().iloc[-1]
            
            return atr / close.iloc[-1]  # Return as percentage of price
        except:
            return 0.02  # Default to 2%
    
    def _calculate_volume_score(self, data: pd.DataFrame) -> float:
        """Calculate volume analysis score"""
        try:
            recent_volume = data['volume'].rolling(window=12).mean().iloc[-1]
            baseline_volume = data['volume'].rolling(window=50).mean().iloc[-1]
            
            volume_ratio = recent_volume / baseline_volume
            return min(100, volume_ratio * 50)  # Scale to 0-100
        except:
            return 50
    
    def _calculate_trend_score(self, data: pd.DataFrame) -> float:
        """Calculate trend strength score"""
        try:
            # Use multiple moving averages
            ma20 = data['close'].rolling(window=20).mean()
            ma50 = data['close'].rolling(window=50).mean()
            
            # Calculate trend alignment
            current_price = data['close'].iloc[-1]
            trend_alignment = (
                (current_price > ma20.iloc[-1]) and
                (ma20.iloc[-1] > ma50.iloc[-1])
            )
            
            # Calculate momentum
            momentum = data['close'].pct_change(periods=10).iloc[-1]
            
            # Combine scores
            trend_score = 50  # Base score
            if trend_alignment:
                trend_score += 25
            trend_score += min(25, max(-25, momentum * 100))
            
            return max(0, min(100, trend_score))
        except:
            return 50
    def _calculate_success_rate(self, data: pd.DataFrame, current_prediction: float) -> float:
        """Calculate success rate of previous predictions"""
        try:
            predictions = []
            actuals = []
            
            for i in range(len(data) - 20, len(data)):
                features = self.create_features(data.iloc[:i])
                X = features.drop(['target'], axis=1)
                X_scaled = self.scaler.transform(X)
                
                model_predictions = []
                for model_type in self.model_types:
                    if model_type == 'lstm':
                        X_lstm, _ = self.prepare_lstm_data(features)
                        pred = self.models[model_type].predict(X_lstm[-1:])
                    else:
                        pred = self.models[model_type].predict(X_scaled[-1:])
                    model_predictions.append(pred[0])
                
                avg_pred = np.mean(model_predictions)
                predictions.append(avg_pred > 0)
                actuals.append(data['returns'].iloc[i] > 0)
            
            return np.mean(np.array(predictions) == np.array(actuals)) * 100
            
        except Exception as e:
            logger.error(f"Error calculating success rate: {str(e)}")
            return 0.0
    
    async def _update_performance_metrics(self, symbol: str, timeframe: str):
        """Update performance metrics for monitoring"""
        try:
            # Get recent signals
            recent_signals = Signal.objects.filter(
                symbol=symbol,
                timeframe=timeframe,
                timestamp__gte=timezone.now() - timedelta(days=7)
            )
            
            # Calculate metrics
            total_signals = len(recent_signals)
            successful_signals = len([s for s in recent_signals if s.is_successful()])
            
            if total_signals > 0:
                success_rate = (successful_signals / total_signals) * 100
            else:
                success_rate = 0
                
            # Update or create metrics
            PerformanceMetrics.objects.update_or_create(
                symbol=symbol,
                timeframe=timeframe,
                defaults={
                    'success_rate': Decimal(str(success_rate)),
                    'total_signals': total_signals,
                    'last_updated': timezone.now()
                }
            )
            
        except Exception as e:
            logger.error(f"Error updating performance metrics: {str(e)}")
    
    async def process_symbol_batch(self, symbols: List[str]):
        """Process a batch of symbols"""
        try:
            for timeframe in self.timeframes:
                for symbol in symbols:
                    if psutil.virtual_memory().available < self.memory_limit:
                        logger.warning("Low memory. Cleaning up...")
                        gc.collect()
                        await asyncio.sleep(1)  # Give system time to recover
                    
                    # Fetch and prepare data
                    data = await self.fetch_historical_data(symbol, timeframe)
                    if data is not None:
                        # Train models if needed
                        await self.optimize_models(data, timeframe)
                        
                        # Generate and save signals
                        signal = await self.generate_signals(symbol, timeframe)
                        if signal:
                            await self.save_signal(signal)
                    
                    await asyncio.sleep(0.1)  # Prevent overwhelming the system
                    
        except Exception as e:
            logger.error(f"Error processing symbol batch: {str(e)}")
    
    async def run(self):
        """Main execution loop with enhanced error handling and resource management"""
        try:
            # Fetch available symbols
            symbols = await self.fetch_symbols()
            if not symbols:
                logger.error("No symbols available for trading")
                return
                
            # Process symbols in batches
            symbol_batches = [symbols[i:i + self.batch_size] 
                            for i in range(0, len(symbols), self.batch_size)]
            
            for batch in symbol_batches:
                await self.process_symbol_batch(batch)
                
                # Memory management after each batch
                gc.collect()
                await asyncio.sleep(1)
                
            logger.info("Trading bot execution completed successfully")
            
        except Exception as e:
            logger.error(f"Error in main execution loop: {str(e)}")
        finally:
            # Cleanup
            gc.collect()
    
    async def start(self):
        """Start the trading bot with error handling"""
        try:
            logger.info("Starting trading bot...")
            while True:
                await self.run()
                
                # Wait before next iteration
                await asyncio.sleep(300)  # 5 minutes between runs
                
        except KeyboardInterrupt:
            logger.info("Trading bot stopped by user")
        except Exception as e:
            logger.error(f"Critical error in trading bot: {str(e)}")
        finally:
            logger.info("Trading bot shutdown complete")
    

    if name == "main": # Initialize and run the bot bot = OptimizedTradingBot(batch_size=5) # Process 5 symbols at a time asyncio.run(bot.start())

  6. Assistant

    The changes you've made to the OptimizedTradingBot show a focus on diverse model usage, better resource management, and a more nuanced approach to signal generation. Here is an evaluation of the key aspects of the current code:

    Improvements and Features

    1. Model Diversity and Weights:

      • You've included a broader array of models (RandomForest, XGBoost, LightGBM, LSTM, ARIMA), which can capture different patterns across time series data due to their varied architectures.
      • The weights for different models in the ensemble now take into account their contributions, which can enhance the robustness of predictions.
    2. Hyperparameter Optimization:

      • Using optuna for hyperparameter tuning allows for more optimized models that can potentially improve predictive performance.
      • The parameter space is efficiently constrained to balance between exploration and resource usage.
    3. Memory and Resource Management:

      • The use of psutil and regular gc.collect() operations helps manage system resources, ensuring that memory is used efficiently.
      • Limiting cores and balancing RAM usage keeps the bot stable even in resource-constrained environments.
    4. Enhanced Technical Analysis:

      • Additional features such as volatility metrics, moving averages, trend measures, and ATR are being used, improving the depth of market understanding.
      • A combination of volume, trend, and success rate scores is used to validate signals, ensuring better quality control.
    5. Signal Validation:

      • Deploying a strategy that includes dynamic thresholds and confidence scores derived from several indicators helps in distinguishing actionable signals from noise.
      • The bot evaluates the success rate of past predictions, factoring in market changes over time.

    Considerations for Further Improvement

    1. Adaptive Parameters and Continuous Learning:

      • Consider incorporating adaptive weight adjustment based on model performance over time to reflect changing market dynamics more effectively.
    2. Real-Time Updates:

      • Implement real-time model updates or reinforcement learning techniques to adapt to market changes immediately.
    3. Backtesting Framework:

      • A systematic backtesting mechanism to validate model changes continuously before going live can further enhance reliability.
    4. Detailed Logging and Monitoring:

      • Ensure detailed logging of decisions taken by the bot, successes, and errors, which are crucial for assessing performance and improving over time.
    5. Scalability Testing:

      • Perform load testing to ensure the bot scales effectively as you add more symbols or operate in more challenging market conditions.

    Final Thoughts

    This version of the OptimizedTradingBot reflects thoughtful improvements in terms of model choice, resource handling, and signal accuracy strategies. Continued refinement in areas of dynamic learning and backtesting will likely further enhance its performance in real trading environments. Overall, the bot is well-structured to be flexible and effective in real-time trading scenarios with diverse market conditions.

  7. User

    import asyncio import logging import os from typing import Dict, List, Tuple, Optional, Any from dataclasses import dataclass from enum import Enum import numpy as np import pandas as pd from datetime import datetime, timedelta from collections import defaultdict import time import warnings from cachetools import TTLCache

    Django imports

    import django from django.utils import timezone from asgiref.sync import sync_to_async from django.db.models import Q

    os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'binancebot.settings') django.setup()

    from .models import Kline, OrderBook, Signal, TradingPair recent_snapshots_cache = TTLCache(maxsize=100, ttl=60)

    Enhanced logging configuration

    logging.basicConfig( level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s - %(funcName)s:%(lineno)d', handlers=[ logging.FileHandler('trading_signals.log'), logging.StreamHandler() ] ) logger = logging.getLogger(name)

    class SignalType(Enum): STRONG_BUY = "Strong Buy" BUY = "Buy" NEUTRAL = "Neutral" SELL = "Sell" STRONG_SELL = "Strong Sell" REVERSAL_BULLISH = "Reversal Bullish" REVERSAL_BEARISH = "Reversal Bearish" NO_SIGNAL = "No Signal"

    @dataclass class MarketRegime: volatility: float trend_strength: float liquidity_score: float regime_type: str confidence: float volume_profile: Dict[str, float] market_efficiency: float regime_change_probability: float

    @dataclass class MarketImpact: price_impact: float volume_impact: float slippage_estimate: float recovery_time: float impact_asymmetry: float market_resilience: float depth_score: float

    @dataclass class SignalMetrics: signal_time: datetime chart_timestamp: datetime symbol: str current_price: float price_24h_change: float signal_type: SignalType stop_loss: float take_profit: float success_rate: float result: str volume_analysis: Dict[str, Any] orderbook_analysis: Dict[str, Any] technical_analysis: Dict[str, Any] market_regime: MarketRegime market_impact: MarketImpact reversal_analysis: Optional[Dict[str, Any]] = None institutional_analysis: Optional[Dict[str, Any]] = None delta_volume_analysis: Optional[Dict[str, Any]] = None vwap_analysis: Optional[Dict[str, Any]] = None signal_confidence: float = 0.0 false_signal_probability: float = 0.0 contributing_factors: List[str] = None # New field to record contributing analyses analysis_type: str = None # New field to record the analysis type

    class DynamicThresholds: """Dynamic thresholds that adapt to changing market conditions"""

    def __init__(self, window_size: int = 100):
        self.window_size = window_size
        self.volume_thresholds: Dict[str, float] = {}
        self.price_thresholds: Dict[str, float] = {}
        self.spread_thresholds: Dict[str, float] = {}
        self.impact_thresholds: Dict[str, float] = {}
            
    async def update_thresholds(self, symbol: str, df: pd.DataFrame, orderbook: Tuple[List, List]) -> None:
        """Update dynamic thresholds based on recent market conditions"""
        try:
            recent_data = df.tail(self.window_size)
            
            # Volume thresholds
            volume_std = recent_data['volume'].std()
            volume_mean = recent_data['volume'].mean()
            self.volume_thresholds[symbol] = {
                'spike': volume_mean + 2 * volume_std,
                'low': volume_mean - volume_std,
                'normal_range': (volume_mean - volume_std, volume_mean + volume_std)
            }
            
            # Price thresholds
            returns = recent_data['close'].pct_change()
            price_std = returns.std()
            self.price_thresholds[symbol] = {
                'volatility': price_std,
                'significant_move': price_std * 2,
                'extreme_move': price_std * 3
            }
            
            # Spread thresholds
            bids, asks = orderbook
            spreads = []
            for i in range(min(10, len(bids), len(asks))):
                bid_price = float(bids[i][0])
                ask_price = float(asks[i][0])
                if bid_price > 0:  # Prevent division by zero
                    spreads.append((ask_price - bid_price) / bid_price)
            
            if spreads:
                spread_mean = np.mean(spreads)
                spread_std = np.std(spreads) if len(spreads) > 1 else 0
                self.spread_thresholds[symbol] = {
                    'normal': spread_mean,
                    'wide': spread_mean + spread_std,
                    'extreme': spread_mean + 2 * spread_std
                }
            
            # Impact thresholds
            impact_samples = self.calculate_impact_samples(orderbook)
            if impact_samples:
                impact_mean = np.mean(impact_samples)
                impact_std = np.std(impact_samples) if len(impact_samples) > 1 else 0
                self.impact_thresholds[symbol] = {
                    'low': impact_mean - impact_std,
                    'medium': impact_mean,
                    'high': impact_mean + impact_std
                }
                
        except Exception as e:
            logger.error(f"Error updating dynamic thresholds: {str(e)}")
            self.reset_thresholds(symbol)
    
    def calculate_impact_samples(self, orderbook: Tuple[List, List]) -> List[float]:
        """Calculate market impact samples for different order sizes"""
        try:
            bids, asks = orderbook
            if not bids or not asks:
                return []
            
            base_size = float(np.mean([float(bid[1]) for bid in bids[:5]]))
            sample_sizes = [base_size * mult for mult in [0.5, 1, 2, 5]]
            
            impacts = []
            for size in sample_sizes:
                bid_impact = self.calculate_size_impact(bids, size)
                ask_impact = self.calculate_size_impact(asks, size)
                if bid_impact is not None and ask_impact is not None:
                    impacts.append((bid_impact + ask_impact) / 2)
            
            return impacts
                
        except Exception as e:
            logger.error(f"Error calculating impact samples: {str(e)}")
            return []
    
    @staticmethod
    def calculate_size_impact(orders: List, size: float) -> Optional[float]:
        """Calculate price impact for a given order size"""
        try:
            if not orders:
                return None
                
            cumulative_size = 0
            initial_price = float(orders[0][0])
            
            for price, order_size in orders:
                price = float(price)
                order_size = float(order_size)
                cumulative_size += order_size
                
                if cumulative_size >= size:
                    return abs(price - initial_price) / initial_price if initial_price > 0 else None
                    
            return None
                
        except Exception as e:
            logger.error(f"Error calculating size impact: {str(e)}")
            return None
    
    def reset_thresholds(self, symbol: str) -> None:
        """Reset thresholds to default values"""
        self.volume_thresholds[symbol] = {
            'spike': 2.0,
            'low': 0.5,
            'normal_range': (0.5, 2.0)
        }
        self.price_thresholds[symbol] = {
            'volatility': 0.02,
            'significant_move': 0.04,
            'extreme_move': 0.06
        }
        self.spread_thresholds[symbol] = {
            'normal': 0.001,
            'wide': 0.002,
            'extreme': 0.005
        }
        self.impact_thresholds[symbol] = {
            'low': 0.001,
            'medium': 0.002,
            'high': 0.005
        }
    

    class EnhancedMarketMicrostructureBot: def init(self): # Core parameters - now using dynamic thresholds self.dynamic_thresholds = DynamicThresholds() self.short_period = 100 self.long_period = 500 self.timeframes = ["1m", "5m", "15m", "1h"]

        # Enhanced market microstructure parameters
        self.liquidity_levels = [0.001, 0.002, 0.005, 0.01, 0.02]
        self.institutional_threshold = 100000.0
        self.max_spread_threshold = 0.002
        self.min_liquidity_requirement = 1000000.0
        
        # Reversal detection parameters
        self.reversal_volume_threshold = 2.0
        self.reversal_price_threshold = 0.02
        self.reversal_confirmation_periods = 3
        self.reversal_momentum_threshold = 0.7
        self.ob_imbalance_threshold = 0.6  # New parameter for orderbook imbalance
        
        # Statistical validation parameters
        self.confidence_level = 0.95
        self.min_sample_size = 30
        self.correlation_threshold = 0.7
        
        # Error handling
        self.error_registry = defaultdict(int)
        self.max_errors = 5
        self.error_timeout = 300  # seconds
    
        # Performance optimization
        self.max_workers = 4
        self.retry_attempts = 3
        self.backoff_factor = 1.5
        self._setup_cache()
    
    def _setup_cache(self) -> None:
        """Setup caching parameters"""
        self.cache_timeouts = {
            'candles': 60,
            'orderbook': 5,
            'trades': 10,
            'analysis': 30
        }
        self.cache_data = {}
    
    @staticmethod
    def get_cache_key(prefix: str, **kwargs) -> str:
        """Generate consistent cache keys"""
        sorted_items = sorted(kwargs.items())
        return f"{prefix}:" + ":".join(f"{k}={v}" for k, v in sorted_items)
    
    async def get_cached_data(self, cache_key: str) -> Optional[Any]:
        """Get data from cache"""
        return self.cache_data.get(cache_key)
    
    async def set_cached_data(self, cache_key: str, data: Any, timeout: int) -> None:
        """Set data in cache"""
        self.cache_data[cache_key] = data
    
    async def handle_error(self, error_key: str, error: Exception) -> bool:
        """Handle errors with exponential backoff and circuit breaker"""
        try:
            self.error_registry[error_key] += 1
            error_count = self.error_registry[error_key]
            
            if error_count >= self.max_errors:
                logger.error(f"Circuit breaker triggered for {error_key}: {str(error)}")
                await self.notify_admin(f"Circuit breaker triggered for {error_key}")
                return False
                
            backoff_time = self.backoff_factor ** (error_count - 1)
            logger.warning(f"Error in {error_key}, attempt {error_count}: {str(error)}")
            await asyncio.sleep(backoff_time)
            
            return True
            
        except Exception as e:
            logger.error(f"Error in error handler: {str(e)}")
            return False
    
    async def notify_admin(self, message: str) -> None:
        """Notify admin of critical issues"""
        logger.critical(message)
        # Implement your notification logic here
    
    @staticmethod
    def validate_data(data: Any, data_type: str) -> bool:
        """Validate data integrity"""
        try:
            if data_type == 'dataframe':
                return isinstance(data, pd.DataFrame) and not data.empty
            elif data_type == 'orderbook':
                return (isinstance(data, tuple) and len(data) == 2 and
                       all(isinstance(x, list) for x in data))
            elif data_type == 'number':
                return isinstance(data, (int, float)) and not np.isnan(data)
            return False
        except Exception:
            return False
    
    async def analyze_market_microstructure(self, pair: str, orderbook: Tuple[List, List]) -> Dict[str, Any]:
        """Enhanced market microstructure analysis"""
        try:
            bids, asks = orderbook
            if not bids or not asks:
                return {}
    
            # Update dynamic thresholds based on current data
            # Fetch recent candles for dynamic threshold update
            timeframe = '1m'  # Using 1-minute data for thresholds
            df = await self.get_candles_df(pair, timeframe)
            await self.dynamic_thresholds.update_thresholds(pair, df, orderbook)
    
            analysis = {
                'liquidity_metrics': await self.analyze_liquidity_metrics(bids, asks),
                'order_flow_metrics': self.analyze_order_flow(bids, asks),
                'institutional_presence': await self.detect_institutional_presence(bids, asks),
                'market_impact': self.calculate_market_impact(bids, asks),
                'microstructure_signals': self.analyze_microstructure_signals(bids, asks),
                'orderbook_pressure': self.calculate_orderbook_pressure(bids, asks)
            }
    
            return analysis
    
        except Exception as e:
            logger.error(f"Error in market microstructure analysis: {str(e)}")
            return {}
    
    async def analyze_liquidity_metrics(self, bids: List, asks: List) -> Dict[str, Any]:
        """Advanced liquidity analysis"""
        liquidity_metrics = {
            'depth_profile': {},
            'resilience_metrics': {},
            'spread_analysis': {},
            'liquidity_score': 0.0
        }
    
        try:
            # Spread analysis
            best_bid = float(bids[0][0])
            best_ask = float(asks[0][0])
            spread = (best_ask - best_bid) / best_bid if best_bid > 0 else 0.0
    
            # Depth profile at different levels
            for level in self.liquidity_levels:
                depth_range = best_bid * level
                bid_depth = sum(float(bid[1]) for bid in bids if float(bid[0]) >= best_bid - depth_range)
                ask_depth = sum(float(ask[1]) for ask in asks if float(ask[0]) <= best_ask + depth_range)
                
                total_depth = bid_depth + ask_depth
                imbalance = (bid_depth - ask_depth) / total_depth if total_depth > 0 else 0
    
                liquidity_metrics['depth_profile'][f"level_{int(level*1000)}bps"] = {
                    'bid_depth': bid_depth,
                    'ask_depth': ask_depth,
                    'imbalance': imbalance
                }
    
            # Liquidity resilience
            liquidity_metrics['resilience_metrics'] = self.calculate_resilience_metrics(bids, asks)
            
            # Spread stability
            liquidity_metrics['spread_analysis'] = {
                'current_spread': spread,
                'spread_stability': self.calculate_spread_stability(bids, asks),
                'effective_spread': self.calculate_effective_spread(bids, asks)
            }
    
            # Overall liquidity score
            liquidity_metrics['liquidity_score'] = self.calculate_liquidity_score(liquidity_metrics)
    
            return liquidity_metrics
    
        except Exception as e:
            logger.error(f"Error in liquidity analysis: {str(e)}")
            return liquidity_metrics
    
    def calculate_resilience_metrics(self, bids: List, asks: List) -> Dict[str, Any]:
        """Calculate market resilience metrics"""
        try:
            best_bid = float(bids[0][0])
            best_ask = float(asks[0][0])
            mid_price = (best_bid + best_ask) / 2
    
            # Calculate order book slope
            bid_slope = self.calculate_order_book_slope(bids, mid_price, side='bid')
            ask_slope = self.calculate_order_book_slope(asks, mid_price, side='ask')
    
            # Calculate price impact for standard volume
            standard_volume = 100000  # Adjust based on your needs
            bid_impact = self.calculate_price_impact(bids, standard_volume, side='bid')
            ask_impact = self.calculate_price_impact(asks, standard_volume, side='ask')
    
            return {
                'bid_slope': bid_slope,
                'ask_slope': ask_slope,
                'bid_impact': bid_impact,
                'ask_impact': ask_impact,
                'symmetry': abs(bid_slope - ask_slope) / max(bid_slope, ask_slope) if max(bid_slope, ask_slope) > 0 else 0.0,
                'recovery_factor': self.estimate_recovery_factor(bids, asks)
            }
    
        except Exception as e:
            logger.error(f"Error calculating resilience metrics: {str(e)}")
            return {}
    
    def calculate_order_book_slope(self, orders: List, mid_price: float, side: str) -> float:
        """Calculate order book slope"""
        try:
            prices = []
            volumes = []
            cumulative_volume = 0
    
            for price, volume in orders[:10]:  # Use top 10 levels
                price = float(price)
                volume = float(volume)
                prices.append(abs(price - mid_price) / mid_price)  # Normalize prices
                cumulative_volume += volume
                volumes.append(cumulative_volume)
    
            if not prices or not volumes:
                return 0.0
    
            # Calculate slope using numpy's polyfit
            slope, _ = np.polyfit(volumes, prices, deg=1)
            return abs(slope)
    
        except Exception as e:
            logger.error(f"Error calculating order book slope: {str(e)}")
            return 0.0
    
    def calculate_price_impact(self, orders: List, volume: float, side: str) -> float:
        """Calculate price impact for given volume"""
        try:
            cumulative_volume = 0
            initial_price = float(orders[0][0])
    
            for price, size in orders:
                price = float(price)
                size = float(size)
                cumulative_volume += size
    
                if cumulative_volume >= volume:
                    # Calculate weighted average price impact
                    return abs(price - initial_price) / initial_price if initial_price > 0 else 0.0
    
            return 1.0  # Maximum impact if order size exceeds available liquidity
    
        except Exception as e:
            logger.error(f"Error calculating price impact: {str(e)}")
            return 1.0
    
    def estimate_recovery_factor(self, bids: List, asks: List) -> float:
        """Estimate market recovery factor"""
        try:
            best_bid = float(bids[0][0])
            best_ask = float(asks[0][0])
            mid_price = (best_bid + best_ask) / 2
    
            # Calculate total available liquidity within 0.5% of mid price
            near_liquidity = sum(float(bid[1]) for bid in bids if float(bid[0]) >= mid_price * 0.995) + \
                           sum(float(ask[1]) for ask in asks if float(ask[0]) <= mid_price * 1.005)
    
            # Calculate total available liquidity within 2% of mid price
            far_liquidity = sum(float(bid[1]) for bid in bids if float(bid[0]) >= mid_price * 0.98) + \
                          sum(float(ask[1]) for ask in asks if float(ask[0]) <= mid_price * 1.02)
    
            if far_liquidity == 0:
                return 0.0
    
            return near_liquidity / far_liquidity
    
        except Exception as e:
            logger.error(f"Error estimating recovery factor: {str(e)}")
            return 0.0
    
    def calculate_liquidity_score(self, metrics: Dict[str, Any]) -> float:
        """Calculate overall liquidity score"""
        try:
            depth_scores = [
                profile['bid_depth'] + profile['ask_depth']
                for profile in metrics['depth_profile'].values()
            ]
            depth_score = np.mean(depth_scores) if depth_scores else 0.0
    
            spread = metrics['spread_analysis']['current_spread']
            liquidity_score = depth_score / (spread + 1e-8)  # Prevent division by zero
            return liquidity_score
    
        except Exception as e:
            logger.error(f"Error calculating liquidity score: {str(e)}")
            return 0.0
    
    def calculate_spread_stability(self, bids: List, asks: List) -> float:
        """Calculate spread stability metric"""
        try:
            spreads = []
            for i in range(min(10, len(bids), len(asks))):
                bid_price = float(bids[i][0])
                ask_price = float(asks[i][0])
                spreads.append((ask_price - bid_price) / bid_price if bid_price > 0 else 0)
                
            return float(np.std(spreads)) if spreads else 1.0
    
        except Exception as e:
            logger.error(f"Error calculating spread stability: {str(e)}")
            return 1.0
    
    def calculate_effective_spread(self, bids: List, asks: List) -> float:
        """Calculate effective spread considering market impact"""
        try:
            if not bids or not asks:
                return 0.0
                
            top_bid = float(bids[0][0])
            top_ask = float(asks[0][0])
            mid_price = (top_bid + top_ask) / 2 if (top_bid + top_ask) > 0 else 0.0
    
            return float((top_ask - top_bid) / mid_price) if mid_price > 0 else 0.0
    
        except Exception as e:
            logger.error(f"Error calculating effective spread: {str(e)}")
            return 0.0
    
    def analyze_order_flow(self, symbol: str) -> Dict[str, Any]:
        """Analyze order flow dynamics"""
        try:
        # Query the most recent snapshot for the given symbol
            current_snapshot = (
                OrderBook.objects
                .filter(symbol=symbol)
                .latest('timestamp')
            )
    
        # Fetch the snapshot before the current one
            previous_snapshot = (
                OrderBook.objects
                .filter(symbol=symbol, timestamp__lt=current_snapshot.timestamp)
                .order_by('-timestamp')
                .first()
            )
    
        # If there's no previous snapshot, we can't compute changes
            if not previous_snapshot:
                logger.warning("Not enough historical data for analysis")
                return {
                    'aggressive_buying': False,
                    'aggressive_selling': False,
                    'bid_size_change': 0.0,
                    'ask_size_change': 0.0
                }
    
        # Calculate bid and ask size changes
            current_bid_size = sum(float(bid['amount']) for bid in current_snapshot.bids)
            current_ask_size = sum(float(ask['amount']) for ask in current_snapshot.asks)
            previous_bid_size = sum(float(bid['amount']) for bid in previous_snapshot.bids)
            previous_ask_size = sum(float(ask['amount']) for ask in previous_snapshot.asks)
    
            bid_size_change = current_bid_size - previous_bid_size
            ask_size_change = current_ask_size - previous_ask_size
    
        # Determine if there's aggressive buying or selling taking place
            aggressive_buying = bid_size_change > 0 and ask_size_change < 0
            aggressive_selling = bid_size_change < 0 and ask_size_change > 0
    
            order_flow_metrics = {
                'aggressive_buying': aggressive_buying,
                'aggressive_selling': aggressive_selling,
                'bid_size_change': bid_size_change,
                'ask_size_change': ask_size_change
            }
    
            return order_flow_metrics
    
        except Exception as e:
            logger.error(f"Error analyzing order flow: {str(e)}")
            return {}
    
    async def detect_institutional_presence(self, bids: List, asks: List) -> Dict[str, Any]:
        """Detect and analyze institutional order flow"""
        try:
            institutional_presence = {
                'large_orders': {'bids': [], 'asks': []},
                'clustering': {'bids': {}, 'asks': {}},
                'iceberg_probability': 0.0,
                'smart_routing_suggestion': None
            }
    
            # Detect large orders
            for side, orders in [('bids', bids), ('asks', asks)]:
                large_orders = []
                price_clusters = defaultdict(float)
    
                for price, size in orders:
                    price, size = float(price), float(size)
                    if size >= self.institutional_threshold:
                        large_orders.append({'price': price, 'size': size})
                    
                    # Round price to analyze clustering
                    price_bucket = round(price, 3)
                    price_clusters[price_bucket] += size
    
                institutional_presence['large_orders'][side] = large_orders
                
                # Analyze clustering
                significant_clusters = {
                    price: volume for price, volume in price_clusters.items()
                    if volume >= self.institutional_threshold
                }
                institutional_presence['clustering'][side] = significant_clusters
    
            # Detect potential iceberg orders
            institutional_presence['iceberg_probability'] = self.detect_iceberg_orders(bids, asks)
    
            # Generate smart routing suggestion
            institutional_presence['smart_routing_suggestion'] = self.generate_smart_routing_suggestion(institutional_presence)
    
            return institutional_presence
    
        except Exception as e:
            logger.error(f"Error detecting institutional presence: {str(e)}")
            return {}
    
    def detect_iceberg_orders(self, bids: List, asks: List) -> float:
        """Detect potential iceberg orders"""
        try:
            # Analyze order size patterns
            bid_sizes = [float(bid[1]) for bid in bids[:5]]  # Top 5 levels
            ask_sizes = [float(ask[1]) for ask in asks[:5]]
    
            # Calculate size variation
            bid_variation = np.std(bid_sizes) / np.mean(bid_sizes) if np.mean(bid_sizes) > 0 else 0
            ask_variation = np.std(ask_sizes) / np.mean(ask_sizes) if np.mean(ask_sizes) > 0 else 0
    
            # High variation might indicate iceberg orders
            iceberg_probability = max(bid_variation, ask_variation) / 2
    
            return min(1.0, iceberg_probability)
    
        except Exception as e:
            logger.error(f"Error detecting iceberg orders: {str(e)}")
            return 0.0
    
    def generate_smart_routing_suggestion(self, institutional_analysis: Dict[str, Any]) -> Dict[str, Any]:
        """Generate smart order routing suggestions"""
        try:
            suggestion = {
                'preferred_venue': 'primary',
                'order_type': 'limit',
                'execution_style': 'normal',
                'urgency': 'medium'
            }
    
            # Analyze institutional presence
            bid_clusters = len(institutional_analysis['clustering']['bids'])
            ask_clusters = len(institutional_analysis['clustering']['asks'])
            large_orders = len(institutional_analysis['large_orders']['bids']) + \
                         len(institutional_analysis['large_orders']['asks'])
    
            # Adjust suggestion based on market conditions
            if large_orders > 5:
                suggestion['execution_style'] = 'careful'
                suggestion['urgency'] = 'low'
                suggestion['order_type'] = 'iceberg'
    
            if bid_clusters > 3 or ask_clusters > 3:
                suggestion['execution_style'] = 'aggressive'
                suggestion['urgency'] = 'high'
    
            if institutional_analysis['iceberg_probability'] > 0.7:
                suggestion['order_type'] = 'peg'
                suggestion['execution_style'] = 'smart'
    
            return suggestion
    
        except Exception as e:
            logger.error(f"Error generating smart routing suggestion: {str(e)}")
            return {}
    
    async def calculate_market_impact(self, bids: List, asks: List) -> Dict[str, Any]:
        """Calculate market impact metrics with detailed analysis"""
        try:
            # Simulate execution of different order sizes
            order_sizes = [10000, 50000, 100000, 500000]  # Example sizes in base currency
            price_impacts = []
            for size in order_sizes:
                buy_impact = self.simulate_order_execution(asks, size, 'buy')
                sell_impact = self.simulate_order_execution(bids, size, 'sell')
                avg_impact = (buy_impact + sell_impact) / 2
                price_impacts.append({'size': size, 'impact': avg_impact})
    
            # Fit a linear regression to estimate impact per unit size
            sizes = np.array([x['size'] for x in price_impacts]).reshape(-1, 1)
            impacts = np.array([x['impact'] for x in price_impacts])
    
            from sklearn.linear_model import LinearRegression
    
            model = LinearRegression()
            model.fit(sizes, impacts)
            impact_per_unit = model.coef_[0]
            intercept = model.intercept_
    
            # Statistical significance
            from scipy import stats
    
            slope, intercept, r_value, p_value, std_err = stats.linregress(sizes.flatten(), impacts)
            significance = 1 - p_value  # Closer to 1 means more significant
    
            market_impact = {
                'impact_per_unit': float(impact_per_unit),
                'intercept': float(intercept),
                'r_squared': r_value ** 2,
                'significance': significance,
                'estimated_impact': price_impacts
            }
    
            return market_impact
    
        except Exception as e:
            logger.error(f"Error calculating market impact: {str(e)}")
            return {}
    
    def simulate_order_execution(self, orders: List, size: float, side: str) -> float:
        """Simulate order execution to estimate price impact"""
        try:
            cumulative_size = 0.0
            initial_price = float(orders[0][0])
            for price, order_size in orders:
                price = float(price)
                order_size = float(order_size)
                cumulative_size += order_size
                if cumulative_size >= size:
                    impact = abs(price - initial_price) / initial_price
                    return impact
            # If order size exceeds available liquidity
            return 1.0  # Maximum impact
        except Exception as e:
            logger.error(f"Error simulating order execution: {str(e)}")
            return 1.0
    
    def analyze_microstructure_signals(self, bids: List, asks: List) -> Dict[str, Any]:
        """Analyze microstructure signals for patterns indicating trading opportunities"""
        try:
            microstructure_signals = {}
    
        # Detect Bid-Ask Bounce
            bid_prices = [float(bid[0]) for bid in bids[:5]]
            ask_prices = [float(ask[0]) for ask in asks[:5]]
            spread = ask_prices[0] - bid_prices[0]
    
            if spread < self.dynamic_thresholds.spread_thresholds.get('normal', 0.001):
                microstructure_signals['bid_ask_bounce'] = True
    
        # Detect Book Pressure
            bid_sizes = [float(bid[1]) for bid in bids[:5]]
            ask_sizes = [float(ask[1]) for ask in asks[:5]]
            bid_volume = sum(bid_sizes)
            ask_volume = sum(ask_sizes)
            pressure = (bid_volume - ask_volume) / (bid_volume + ask_volume)
    
            if pressure > 0.5:
                microstructure_signals['buy_pressure'] = True
            elif pressure < -0.5:
                microstructure_signals['sell_pressure'] = True
    
        # Detect Quote Stuffing
            if self.detect_quote_stuffing(bids, asks):
                microstructure_signals['quote_stuffing_detected'] = True
    
        # Detect Layers (Spoofing)
            if self.detect_layers(bids, asks):
                microstructure_signals['layers_detected'] = True
    
            return microstructure_signals
    
        except Exception as e:
            logger.error(f"Error analyzing microstructure signals: {str(e)}")
            return {}
    
    def detect_quote_stuffing(self, symbol: str, lookback_period: int = 1, order_threshold: int = 100) -> bool:
        """
        Detect quote stuffing by rapid order book updates.
    
        :param symbol: The trading symbol to analyze.
        :param lookback_period: Number of minutes to look back for analyzing frequent updates.
        :param order_threshold: The number of order changes considered indicative of quote stuffing.
        :return: True if quote stuffing is detected, otherwise False.
        """
        try:
            # Calculate the timestamp limit for the lookback period
            now = timezone.now()
            lookback_time = now - timedelta(minutes=lookback_period)
    
        # Utilize cached data when possible
            cache_key = f"{symbol}_{lookback_time.isoformat()}"
            if cache_key in recent_snapshots_cache:
                recent_snapshots = recent_snapshots_cache[cache_key]
            else:
            # Query recent order book snapshots within the lookback period
                recent_snapshots = list(OrderBook.objects.filter(
                    symbol=symbol,
                    timestamp__gte=lookback_time
                ).order_by('timestamp'))
                recent_snapshots_cache[cache_key] = recent_snapshots  # Update cache
    
        # If there are too few snapshots, quote stuffing is not detectable
            if len(recent_snapshots) < 2:
                return False
    
        # Initialize order update count
            order_update_count = 0
    
        # Analyze snapshots for characteristics of quote stuffing
            for i in range(1, len(recent_snapshots)):
                previous_snapshot = recent_snapshots[i - 1]
                current_snapshot = recent_snapshots[i]
    
            # Calculate changes in price levels and sizes
                bid_changes = set(map(tuple, current_snapshot.bids)) ^ set(map(tuple, previous_snapshot.bids))
                ask_changes = set(map(tuple, current_snapshot.asks)) ^ set(map(tuple, previous_snapshot.asks))
    
            # Detect rapid significant changes
                order_update_count += len(bid_changes) + len(ask_changes)
    
            # Immediately return True if the threshold is surpassed
                if order_update_count > order_threshold:
                    return True
    
            return False
    
        except Exception as e:
            logger.error(f"Error detecting quote stuffing: {str(e)}")
            return False
    
    def detect_layers(self, bids: List, asks: List) -> bool:
        """Detect layering or spoofing activities."""
        try:
            # Check for unusually large sizes at different price levels
            bid_sizes = [float(bid[1]) for bid in bids]
            ask_sizes = [float(ask[1]) for ask in asks]
    
            threshold = np.mean(bid_sizes + ask_sizes) + 2 * np.std(bid_sizes + ask_sizes)
    
            for i in range(1, len(bids)):
                if float(bids[i][1]) > threshold:
                    return True
            for i in range(1, len(asks)):
                if float(asks[i][1]) > threshold:
                    return True
            return False
        except Exception as e:
            logger.error(f"Error detecting layers: {str(e)}")
            return False
    
    def calculate_orderbook_pressure(self, bids: List, asks: List) -> Dict[str, Any]:
        """Calculate orderbook pressure"""
        try:
            bid_volume = sum(float(bid[1]) for bid in bids[:10])
            ask_volume = sum(float(ask[1]) for ask in asks[:10])
            total_volume = bid_volume + ask_volume
    
            pressure = (bid_volume - ask_volume) / total_volume if total_volume > 0 else 0.0
    
            return {
                'pressure': pressure,
                'side': 'buy' if pressure > 0 else 'sell' if pressure < 0 else 'neutral'
            }
    
        except Exception as e:
            logger.error(f"Error calculating orderbook pressure: {str(e)}")
            return {}
    
    async def analyze_institutional_activity(self, orderbook: Tuple[List, List], volume_analysis: Dict[str, Any]) -> Dict[str, Any]:
        """Analyze institutional trading activity"""
        try:
            bids, asks = orderbook
            institutional_analysis = await self.detect_institutional_presence(bids, asks)
            # Additional analysis can be added here
            return institutional_analysis
    
        except Exception as e:
            logger.error(f"Error analyzing institutional activity: {str(e)}")
            return {}
    
    def calculate_institutional_score(self, institutional_analysis: Dict[str, Any]) -> float:
        """Calculate institutional score"""
        try:
            # Placeholder implementation
            large_orders_count = len(institutional_analysis['large_orders']['bids']) + \
                                 len(institutional_analysis['large_orders']['asks'])
            iceberg_probability = institutional_analysis.get('iceberg_probability', 0.0)
            institutional_score = (large_orders_count * 0.1) + (iceberg_probability * 0.9)
            return min(institutional_score, 1.0)
    
        except Exception as e:
            logger.error(f"Error calculating institutional score: {str(e)}")
            return 0.0
    
    async def generate_signal(self, pair: str, timeframe: str) -> Tuple[SignalType, Optional[SignalMetrics]]:
        """Generate comprehensive trading signal"""
        try:
            # Fetch required data
            df = await self.get_candles_df(pair, timeframe)
            if df.empty:
                return SignalType.NO_SIGNAL, None
    
            orderbook = await self.get_orderbook(pair)
            if not orderbook[0] or not orderbook[1]:
                return SignalType.NO_SIGNAL, None
    
            # Perform comprehensive analysis
            current_price = float(df['close'].iloc[-1])
    
            # Adjust calculation for price_24h_ago
            now = df.index[-1]
            price_24h_ago_time = now - timedelta(hours=24)
            df_24h_ago = df[df.index <= price_24h_ago_time]
            if not df_24h_ago.empty:
                price_24h_ago = float(df_24h_ago['close'].iloc[-1])
            else:
                price_24h_ago = float(df['close'].iloc[0])
            price_24h_change = ((current_price - price_24h_ago) / price_24h_ago)
    
            # Volume analysis
            volume_analysis = await self.analyze_volume_metrics(df)
            # Orderbook analysis
            orderbook_analysis = await self.analyze_market_microstructure(pair, orderbook)
            # Market regime
            market_regime = await self.detect_market_regime(df, orderbook_analysis)
            # Market impact
            market_impact = await self.calculate_detailed_market_impact(orderbook)
            # Reversal analysis
            reversal_analysis = await self.detect_trend_reversal(df, orderbook, volume_analysis)
            # Institutional analysis
            institutional_analysis = await self.analyze_institutional_activity(orderbook, volume_analysis)
    
            # Generate signal type
            signal_type, contributing_factors = await self.determine_signal_type(
                df, orderbook_analysis, volume_analysis, market_regime,
                reversal_analysis, institutional_analysis
            )
            if signal_type == SignalType.NO_SIGNAL:
                return signal_type, None
    
            # Calculate risk levels
            stop_loss, take_profit = await self.calculate_risk_levels(
                signal_type, current_price, orderbook_analysis, market_impact
            )
    
            # Calculate success rate
            success_rate = await self.calculate_success_probability(
                signal_type, volume_analysis, orderbook_analysis,
                market_regime, institutional_analysis
            )
    
            # Prepare signal metrics
            metrics = SignalMetrics(
                signal_time=timezone.now(),
                chart_timestamp=df.index[-1],
                symbol=pair,
                current_price=current_price,
                price_24h_change=price_24h_change * 100,
                signal_type=signal_type,
                stop_loss=stop_loss,
                take_profit=take_profit,
                success_rate=success_rate * 100,  # Convert to percentage
                result="Pending",
                volume_analysis=volume_analysis,
                orderbook_analysis=orderbook_analysis,
                technical_analysis={},  # For now, empty as we are focusing on orderbook
                market_regime=market_regime,
                market_impact=market_impact,
                reversal_analysis=reversal_analysis,
                institutional_analysis=institutional_analysis,
                contributing_factors=contributing_factors
            )
    
            # Save signal to database
            await self.save_signal(metrics)
    
            return signal_type, metrics
        except Exception as e:
            logger.error(f"Error in generate_signal: {str(e)}")
            return SignalType.NO_SIGNAL, None
    
    async def analyze_volume_metrics(self, df: pd.DataFrame) -> Dict[str, Any]:
        """Analyze volume patterns and characteristics"""
        try:
            recent_volume = df['volume'].tail(self.short_period)
            volume_sma = recent_volume.mean()
            volume_std = recent_volume.std()
            volume_zscore = (recent_volume.iloc[-1] - volume_sma) / volume_std if volume_std > 0 else 0.0
    
            return {
                'current_volume': float(recent_volume.iloc[-1]),
                'volume_sma': float(volume_sma),
                'volume_std': float(volume_std),
                'volume_zscore': float(volume_zscore),
                'volume_trend': self.calculate_volume_trend(recent_volume),
                'volume_concentration': self.analyze_volume_concentration(df),
                'buying_pressure': self.calculate_buying_pressure(df),
                'volume_profile': self.analyze_volume_profile(df)
            }
        except Exception as e:
            logger.error(f"Error in volume analysis: {str(e)}")
            return {}
    
    def calculate_volume_trend(self, volumes: pd.Series) -> str:
        """Analyze volume trend"""
        try:
            volume_sma = volumes.rolling(window=20).mean()
            current_vol = volumes.iloc[-1]
            avg_vol = volume_sma.iloc[-1] if volume_sma.iloc[-1] > 0 else volumes.mean()
    
            if current_vol > avg_vol * 2:
                return "strong_increasing"
            elif current_vol > avg_vol * 1.5:
                return "increasing"
            elif current_vol < avg_vol * 0.5:
                return "strong_decreasing"
            elif current_vol < avg_vol * 0.75:
                return "decreasing"
            else:
                return "stable"
    
        except Exception as e:
            logger.error(f"Error calculating volume trend: {str(e)}")
            return "unknown"
    
    def analyze_volume_concentration(self, df: pd.DataFrame) -> Dict[str, Any]:
        """Analyze volume concentration patterns"""
        try:
            recent_df = df.tail(100)
            price_range = recent_df['close'].max() - recent_df['close'].min()
            bins = np.linspace(recent_df['close'].min(), recent_df['close'].max(), num=10)
            volume_price_levels = pd.cut(recent_df['close'], bins=bins, include_lowest=True)
            volume_concentration = recent_df.groupby(volume_price_levels)['volume'].sum()
    
            return {
                'high_volume_levels': volume_concentration.nlargest(3).index.astype(str).tolist(),
                'concentration_ratio': float(volume_concentration.max() / volume_concentration.sum()) if volume_concentration.sum() > 0 else 0.0,
                'distribution': volume_concentration.to_dict()
            }
    
        except Exception as e:
            logger.error(f"Error analyzing volume concentration: {str(e)}")
            return {}
    
    def calculate_buying_pressure(self, df: pd.DataFrame) -> float:
        """Calculate buying pressure based on volume and price movement"""
        try:
            recent_df = df.tail(20)
            up_volume = recent_df[recent_df['close'] > recent_df['open']]['volume'].sum()
            down_volume = recent_df[recent_df['close'] < recent_df['open']]['volume'].sum()
            total_volume = up_volume + down_volume
    
            if total_volume == 0:
                return 0.5
    
            return float(up_volume / total_volume)
    
        except Exception as e:
            logger.error(f"Error calculating buying pressure: {str(e)}")
            return 0.5
    
    def analyze_volume_profile(self, df: pd.DataFrame) -> Dict[str, Any]:
        """Analyze volume profile characteristics"""
        try:
            recent_df = df.tail(100)
    
            profile = {
                'avg_volume': float(recent_df['volume'].mean()),
                'vol_std': float(recent_df['volume'].std()),
                'volume_pods': {},
                'volume_trend_strength': 0.0
            }
    
            # Calculate volume Points of Distribution
            price_levels = pd.qcut(recent_df['close'], q=10, duplicates='drop')
            volume_pods = recent_df.groupby(price_levels)['volume'].agg(['sum', 'count'])
            profile['volume_pods'] = volume_pods.to_dict()
    
            # Calculate volume trend strength
            volume_sma = recent_df['volume'].rolling(window=20).mean()
            if not volume_sma.empty and volume_sma.iloc[-1] > 0:
                volume_trend = recent_df['volume'].iloc[-1] / volume_sma.iloc[-1]
            else:
                volume_trend = 1.0
            profile['volume_trend_strength'] = float(volume_trend)
    
            return profile
    
        except Exception as e:
            logger.error(f"Error analyzing volume profile: {str(e)}")
            return {}
    
    async def detect_market_regime(self, df: pd.DataFrame, orderbook_analysis: Dict[str, Any]) -> MarketRegime:
        """Detect current market regime"""
        try:
            # Calculate volatility
            returns = df['close'].pct_change()
            volatility = returns.std() * np.sqrt(len(df))
    
            # Calculate trend strength
            price_sma = df['close'].rolling(window=20).mean()
            if price_sma.iloc[-1] > 0:
                trend_strength = abs(df['close'].iloc[-1] - price_sma.iloc[-1]) / price_sma.iloc[-1]
            else:
                trend_strength = 0.0
    
            # Get liquidity score
            liquidity_score = orderbook_analysis.get('liquidity_metrics', {}).get('liquidity_score', 0.0)
    
            # Determine regime type
            if volatility > 0.02 and liquidity_score < 0.5:
                regime_type = "high_volatility_low_liquidity"
                confidence = 0.8
            elif volatility > 0.02:
                regime_type = "high_volatility"
                confidence = 0.85
            elif liquidity_score < 0.5:
                regime_type = "low_liquidity"
                confidence = 0.75
            elif trend_strength > 0.02:
                regime_type = "trending"
                confidence = 0.9
            else:
                regime_type = "normal"
                confidence = 0.7
    
            market_regime = MarketRegime(
                volatility=float(volatility),
                trend_strength=float(trend_strength),
                liquidity_score=float(liquidity_score),
                regime_type=regime_type,
                confidence=confidence,
                volume_profile={},
                market_efficiency=0.0,
                regime_change_probability=0.0
            )
    
            return market_regime
    
        except Exception as e:
            logger.error(f"Error detecting market regime: {str(e)}")
            return MarketRegime(0, 0, 0, "unknown", 0, {}, 0, 0)
    
    async def calculate_detailed_market_impact(self, orderbook: Tuple[List, List]) -> MarketImpact:
        """Calculate detailed market impact metrics"""
        try:
            bids, asks = orderbook
            if not bids or not asks:
                return MarketImpact(0, 0, 0, 0, 0, 0, 0)
    
            # Calculate immediate price impact for standard size
            standard_size = 10000  # Adjust based on your needs
            bid_impact = self.calculate_price_impact(bids, standard_size, 'bid')
            ask_impact = self.calculate_price_impact(asks, standard_size, 'ask')
            price_impact = (bid_impact + ask_impact) / 2
    
            # Calculate volume impact
            bid_volume = sum(float(bid[1]) for bid in bids[:10])
            ask_volume = sum(float(ask[1]) for ask in asks[:10])
            total_volume = bid_volume + ask_volume
            volume_impact = standard_size / total_volume if total_volume > 0 else 1.0
    
            # Estimate slippage
            slippage_estimate = price_impact * (1 + volume_impact)
    
            # Estimate recovery time based on volume and impact
            recovery_time = self.estimate_recovery_time(slippage_estimate, volume_impact)
    
            market_impact = MarketImpact(
                price_impact=float(price_impact),
                volume_impact=float(volume_impact),
                slippage_estimate=float(slippage_estimate),
                recovery_time=float(recovery_time),
                impact_asymmetry=0.0,
                market_resilience=0.0,
                depth_score=0.0
            )
    
            return market_impact
    
        except Exception as e:
            logger.error(f"Error calculating market impact: {str(e)}")
            return MarketImpact(0, 0, 0, 0, 0, 0, 0)
    
    def estimate_recovery_time(self, slippage: float, volume_impact: float) -> float:
        """Estimate market recovery time in minutes"""
        try:
            base_recovery = 5  # Base recovery time in minutes
            impact_multiplier = 1 + (slippage * 10)  # Higher slippage = longer recovery
            volume_multiplier = 1 + (volume_impact * 5)  # Higher volume impact = longer recovery
    
            return base_recovery * impact_multiplier * volume_multiplier
    
        except Exception as e:
            logger.error(f"Error estimating recovery time: {str(e)}")
            return 5.0
    
    async def detect_trend_reversal(
        self, df: pd.DataFrame, orderbook: Tuple[List, List],
        volume_analysis: Dict[str, Any]
    ) -> Dict[str, Any]:
        """Detect potential trend reversals"""
        try:
            bids, asks = orderbook
            current_price = float(df['close'].iloc[-1])
    
            analysis = {
                'reversal_detected': False,
                'direction': None,
                'confidence': 0,
                'supporting_factors': []
            }
    
            # Volume surge check
            volume_zscore = volume_analysis.get('volume_zscore', 0.0)
            if abs(volume_zscore) > self.reversal_volume_threshold:
                analysis['supporting_factors'].append('volume_surge')
    
            # Orderbook imbalance check
            bid_volume = sum(float(bid[1]) for bid in bids[:10])
            ask_volume = sum(float(ask[1]) for ask in asks[:10])
            ob_imbalance = (bid_volume - ask_volume) / (bid_volume + ask_volume) if (bid_volume + ask_volume) > 0 else 0.0
    
            if abs(ob_imbalance) > self.ob_imbalance_threshold:
                analysis['supporting_factors'].append('orderbook_imbalance')
    
            # Price movement check
            recent_returns = df['close'].pct_change().tail(self.reversal_confirmation_periods)
            price_change = recent_returns.sum()
    
            if abs(price_change) > self.reversal_price_threshold:
                analysis['supporting_factors'].append('price_movement')
    
            # Determine reversal
            if len(analysis['supporting_factors']) >= 2:
                analysis['reversal_detected'] = True
                analysis['direction'] = 'bullish' if ob_imbalance > 0 else 'bearish'
                analysis['confidence'] = min(0.9, 0.6 + len(analysis['supporting_factors']) * 0.1)
    
            return analysis
    
        except Exception as e:
            logger.error(f"Error detecting trend reversal: {str(e)}")
            return {'reversal_detected': False}
    
    async def determine_signal_type(
        self, df: pd.DataFrame, orderbook_analysis: Dict[str, Any],
        volume_analysis: Dict[str, Any], market_regime: MarketRegime,
        reversal_analysis: Dict[str, Any], institutional_analysis: Dict[str, Any]
    ) -> Tuple[SignalType, List[str]]:
        """Determine signal type based on comprehensive analysis"""
        try:
            contributing_factors = []
    
            # Calculate component scores
            volume_score = self.calculate_volume_score(volume_analysis)
            orderbook_score = self.calculate_orderbook_score(orderbook_analysis)
            institutional_score = self.calculate_institutional_score(institutional_analysis)
    
            # Combined weighted score
            total_score = (
                volume_score * 0.4 +
                orderbook_score * 0.4 +
                institutional_score * 0.2
            )
    
            # Check for reversals first
            if reversal_analysis.get('reversal_detected'):
                if reversal_analysis['direction'] == 'bullish':
                    contributing_factors.append('reversal_bullish')
                    return SignalType.REVERSAL_BULLISH, contributing_factors
                contributing_factors.append('reversal_bearish')
                return SignalType.REVERSAL_BEARISH, contributing_factors
    
            # Check for range-bound market
            if self.is_range_bound(df, market_regime):
                contributing_factors.append('range_bound')
                return SignalType.NEUTRAL, contributing_factors
    
            # Determine signal based on total score
            if total_score > 0.8:
                contributing_factors.append('strong_buy')
                return SignalType.STRONG_BUY, contributing_factors
            elif total_score > 0.6:
                contributing_factors.append('buy')
                return SignalType.BUY, contributing_factors
            elif total_score < -0.8:
                contributing_factors.append('strong_sell')
                return SignalType.STRONG_SELL, contributing_factors
            elif total_score < -0.6:
                contributing_factors.append('sell')
                return SignalType.SELL, contributing_factors
    
            return SignalType.NO_SIGNAL, contributing_factors
    
        except Exception as e:
            logger.error(f"Error determining signal type: {str(e)}")
            return SignalType.NO_SIGNAL, []
    
    def calculate_volume_score(self, volume_analysis: Dict[str, Any]) -> float:
        """Calculate volume score"""
        try:
            volume_zscore = volume_analysis.get('volume_zscore', 0.0)
            volume_trend = volume_analysis.get('volume_trend', 'stable')
            buying_pressure = volume_analysis.get('buying_pressure', 0.5)
    
            if volume_trend in ['strong_increasing', 'increasing']:
                trend_score = 1.0
            elif volume_trend in ['strong_decreasing', 'decreasing']:
                trend_score = -1.0
            else:
                trend_score = 0.0
    
            volume_score = volume_zscore / 3 + trend_score * 0.5 + (buying_pressure - 0.5) * 2
    
            return max(min(volume_score, 1.0), -1.0)
    
        except Exception as e:
            logger.error(f"Error calculating volume score: {str(e)}")
            return 0.0
    
    def calculate_orderbook_score(self, orderbook_analysis: Dict[str, Any]) -> float:
        """Calculate orderbook score"""
        try:
            ob_pressure = orderbook_analysis.get('orderbook_pressure', {}).get('pressure', 0.0)
            liquidity_score = orderbook_analysis.get('liquidity_metrics', {}).get('liquidity_score', 0.0)
            microstructure_signals = orderbook_analysis.get('microstructure_signals', {})
    
            # Calculate contribution from microstructure signals
            microstructure_score = 0.0
    
        # Adjust the microstructure_score based on detected signals
            if microstructure_signals.get('bid_ask_bounce'):
                microstructure_score += 0.1  # Assign an appropriate value
    
            if microstructure_signals.get('buy_pressure'):
                microstructure_score += 0.1
    
            if microstructure_signals.get('sell_pressure'):
                microstructure_score -= 0.1
    
            if microstructure_signals.get('quote_stuffing_detected'):
                microstructure_score -= 0.2
    
            if microstructure_signals.get('layers_detected'):
                microstructure_score -= 0.2
    
        # Normalize microstructure_score to a reasonable range if needed
            microstructure_score = max(min(microstructure_score, 0.5), -0.5)
    
        # Calculate the final orderbook score with the contributions
            orderbook_score = ob_pressure * 0.5 + liquidity_score * 0.3 + microstructure_score * 0.2
    
        # Ensure the score is between -1.0 and 1.0
            return max(min(orderbook_score, 1.0), -1.0)
    
    
        except Exception as e:
            logger.error(f"Error calculating orderbook score: {str(e)}")
            return 0.0
    
    async def calculate_success_probability(
        self, signal_type: SignalType, volume_analysis: Dict[str, Any],
        orderbook_analysis: Dict[str, Any], market_regime: MarketRegime,
        institutional_analysis: Dict[str, Any]
    ) -> float:
        """Calculate probability of signal success"""
        try:
            # Base probability from volume analysis
            volume_probability = min(1.0, volume_analysis.get('volume_zscore', 0.0) / 3)
    
            # Orderbook confidence
            ob_confidence = orderbook_analysis.get('liquidity_metrics', {}).get('liquidity_score', 0.0)
    
            # Institutional confidence
            inst_confidence = institutional_analysis.get('iceberg_probability', 0.0)
    
            # Market regime adjustment
            regime_multiplier = 1.0
            if market_regime.regime_type == "high_volatility":
                regime_multiplier = 0.8
            elif market_regime.regime_type == "low_liquidity":
                regime_multiplier = 0.7
    
            # Combined probability
            base_probability = (
                volume_probability * 0.4 +
                ob_confidence * 0.4 +
                inst_confidence * 0.2
            ) * regime_multiplier
    
            # Adjust based on signal type
            if signal_type in [SignalType.STRONG_BUY, SignalType.STRONG_SELL]:
                base_probability *= 1.2
            elif signal_type in [SignalType.REVERSAL_BULLISH, SignalType.REVERSAL_BEARISH]:
                base_probability *= 0.9
    
            return min(0.95, max(0.05, base_probability))
    
        except Exception as e:
            logger.error(f"Error calculating success probability: {str(e)}")
            return 0.5
    
    async def calculate_risk_levels(
        self, signal_type: SignalType, current_price: float,
        orderbook_analysis: Dict[str, Any], market_impact: MarketImpact
    ) -> Tuple[float, float]:
        """Calculate stop loss and take profit levels"""
        try:
            # Base risk percentages
            if signal_type in [SignalType.STRONG_BUY, SignalType.STRONG_SELL]:
                sl_percent = 0.02
                tp_percent = 0.06
            elif signal_type in [SignalType.BUY, SignalType.SELL]:
                sl_percent = 0.015
                tp_percent = 0.045
            else:
                sl_percent = 0.025
                tp_percent = 0.05
    
            # Adjust based on market impact
            sl_percent *= (1 + market_impact.slippage_estimate)
            tp_percent *= (1 + market_impact.slippage_estimate)
    
            # Calculate levels
            if signal_type in [SignalType.STRONG_BUY, SignalType.BUY, SignalType.REVERSAL_BULLISH]:
                stop_loss = current_price * (1 - sl_percent)
                take_profit = current_price * (1 + tp_percent)
            else:
                stop_loss = current_price * (1 + sl_percent)
                take_profit = current_price * (1 - tp_percent)
    
            return stop_loss, take_profit
    
        except Exception as e:
            logger.error(f"Error calculating risk levels: {str(e)}")
            return current_price * 0.98, current_price * 1.02
    
    def is_range_bound(self, df: pd.DataFrame, market_regime: MarketRegime) -> bool:
        """Determine if market is range-bound"""
        try:
            # Check volatility
            if market_regime.volatility > 0.02:
                return False
    
            recent_df = df.tail(100)
            price_range = (recent_df['high'].max() - recent_df['low'].min()) / recent_df['close'].mean()
    
            # Check if price is contained within a narrow range
            if price_range > 0.03:
                return False
    
            # Check for strong trends
            if market_regime.trend_strength > 0.02:
                return False
    
            # Check volume stability
            volume_stability = recent_df['volume'].std() / recent_df['volume'].mean() if recent_df['volume'].mean() > 0 else 0.0
            if volume_stability > 0.5:
                return False
    
            return True
    
        except Exception as e:
            logger.error(f"Error checking range-bound market: {str(e)}")
            return False
    
    async def save_signal(self, metrics: SignalMetrics) -> None:
        """Save signal to database"""
        try:
            signal = Signal(
                timestamp=metrics.signal_time,
                chart_timestamp=metrics.chart_timestamp,
                symbol=metrics.symbol,
                price=metrics.current_price,
                price_24h_change=metrics.price_24h_change,
                signal=metrics.signal_type.value,
                stop_loss=metrics.stop_loss,
                take_profit=metrics.take_profit,
                success_rate=metrics.success_rate,
                result=metrics.result,
                analysis_details={
                    'volume_analysis': metrics.volume_analysis,
                    'orderbook_analysis': metrics.orderbook_analysis,
                    'contributing_factors': metrics.contributing_factors,
                    'market_regime': {
                        'volatility': metrics.market_regime.volatility,
                        'trend_strength': metrics.market_regime.trend_strength,
                        'liquidity_score': metrics.market_regime.liquidity_score,
                        'regime_type': metrics.market_regime.regime_type,
                        'confidence': metrics.market_regime.confidence
                    },
                    'market_impact': {
                        'price_impact': metrics.market_impact.price_impact,
                        'slippage_estimate': metrics.market_impact.slippage_estimate,
                        'volume_impact': metrics.market_impact.volume_impact
                    },
                    'reversal_analysis': metrics.reversal_analysis,
                    'institutional_analysis': metrics.institutional_analysis
    
                }
            )
    
            await sync_to_async(signal.save)()
    
        except Exception as e:
            logger.error(f"Error saving signal: {str(e)}")
    
    async def get_candles_df(self, pair: str, timeframe: str) -> pd.DataFrame:
        """Fetch candlestick data from database"""
        try:
            # Use cache
            cache_key = self.get_cache_key('candles', pair=pair, timeframe=timeframe)
            cached_df = await self.get_cached_data(cache_key)
            if cached_df is not None:
                return cached_df
    
            klines = await sync_to_async(list)(
                Kline.objects.filter(
                    symbol=pair,
                    timeframe=timeframe
                ).order_by('timestamp')
                .values('timestamp', 'open', 'high', 'low', 'close', 'volume')
            )
    
            if not klines:
                return pd.DataFrame()
    
            df = pd.DataFrame(klines)
            df['timestamp'] = pd.to_datetime(df['timestamp'])
            df.set_index('timestamp', inplace=True)
            await self.set_cached_data(cache_key, df, self.cache_timeouts['candles'])
            return df
    
        except Exception as e:
            logger.error(f"Error fetching candles: {str(e)}")
            return pd.DataFrame()
    
    async def get_orderbook(self, pair: str) -> Tuple[List, List]:
        """Fetch orderbook data from database"""
        try:
            # Use cache
            cache_key = self.get_cache_key('orderbook', pair=pair)
            cached_orderbook = await self.get_cached_data(cache_key)
            if cached_orderbook is not None:
                return cached_orderbook
    
            orderbook = await sync_to_async(OrderBook.objects.filter(symbol=pair).first)()
            if not orderbook:
                return [], []
    
            bids = orderbook.bids
            asks = orderbook.asks
            await self.set_cached_data(cache_key, (bids, asks), self.cache_timeouts['orderbook'])
            return bids, asks
    
        except Exception as e:
            logger.error(f"Error fetching orderbook: {str(e)}")
            return [], []
    
    async def run(self):
        """Main method to run the bot for all symbols and timeframes."""
        try:
            symbols = await self.get_symbols()
            tasks = []
            for symbol in symbols:
                for timeframe in self.timeframes:
                    tasks.append(self.generate_signal(symbol, timeframe))
            await asyncio.gather(*tasks)
        except Exception as e:
            logger.error(f"Error running bot: {str(e)}")
    
    async def get_symbols(self) -> List[str]:
        """Fetch the list of trading pairs (symbols)."""
        trading_pairs = await sync_to_async(list)(TradingPair.objects.values_list('symbol', flat=True))
        return trading_pairs
    

    if name == 'main': bot = EnhancedMarketMicrostructureBot() asyncio.run(bot.run())

    Can you analyze the above code to debug it. Tell me more about institutional_presence and institutional_analysis and if they are all implemented as it should. Debug for any incomplete functions or erroneous methods. And lastly, write def calculate_institutional_score function correctly and replace the placeholder with fully implemented logic. If something needs to be modified you need to write the whole method for tthe function.
    
  8. Assistant

    Analyzing the code involves examining the flow, identifying any erroneous or incomplete logic, and ensuring functionalities are implemented as intended. Here’s a focused analysis of the institutional_presence and institutional_analysis functionalities, including a correction for the calculate_institutional_score function.

    Institutional Presence and Analysis

    • Purpose: The institutional_presence and institutional_analysis aim to detect significant trading behaviors, such as large orders or iceberg orders, often associated with institutional traders.

    • Key Methods:

      1. detect_institutional_presence: Identifies large orders and clusters them, determines iceberg possibilities, and provides smart routing suggestions based on identified patterns.

      2. detect_iceberg_orders: Estimates the presence of iceberg orders by assessing volume variations at the top order book levels.

      3. generate_smart_routing_suggestion: Provides trading strategy recommendations given the detected presence of large orders or iceberg probabilities.

    Debugging and Enhancements

    Incomplete and Erroneous Elements

    1. Logic and Flow Errors:

      • The detect_institutional_presence method is well-defined, but the application of thresholds and interpretation logic should be reviewed to align with actual trading data dynamics.
      • The function detect_iceberg_orders assumes that variation alone indicates iceberg orders, which might not account for context or other supporting signals.
    2. Improving calculate_institutional_score:

      • Currently, this function contains a placeholder that only considers large order counts and iceberg probability. The revised version should incorporate more nuanced factors like clustering significance and potential market impact.

    Complete Implementation of calculate_institutional_score

    def calculate_institutional_score(self, institutional_analysis: Dict[str, Any]) -> float:
        """Calculate the institutional score based on order characteristics and clustering."""
        try:
            # Extract necessary information
            large_bids = institutional_analysis['large_orders']['bids']
            large_asks = institutional_analysis['large_orders']['asks']
            iceberg_prob = institutional_analysis.get('iceberg_probability', 0.0)
            bid_clusters = institutional_analysis['clustering']['bids']
            ask_clusters = institutional_analysis['clustering']['asks']
    
            # Define weights for scoring
            large_order_weight = 0.3
            iceberg_weight = 0.3
            clustering_weight = 0.2
            market_impact_weight = 0.2
    
            # Calculate scores
            large_order_score = (len(large_bids) + len(large_asks)) / 10.0  # Example normalization
            clustering_score = (len(bid_clusters) + len(ask_clusters)) / 10.0
    
            # Example market impact calculation based on known impact factors
            market_impact_score = min(1.0, 0.1 * (len(large_bids) + len(large_asks)))
    
            # Total score computation
            institutional_score = (
                large_order_score * large_order_weight +
                iceberg_prob * iceberg_weight +
                clustering_score * clustering_weight +
                market_impact_score * market_impact_weight
            )
    
            # Normalize institutional_score to a range of 0 - 1
            return min(institutional_score, 1.0)
    
        except Exception as e:
            logger.error(f"Error calculating institutional score: {str(e)}")
            return 0.0
    

    Suggestions for Improvements and Completeness

    • Validation: All significant operations like estimates of iceberg probability or clustering behavior should be validated against actual data patterns observed historically or via backtesting, ensuring their reliability.

    • Comprehensive Logging: Enhance logging for critical points within each function, such as calculated scores or decisions made, providing better traceability.

    • Dynamic Calibration: Consider incorporating adaptive methods that adjust weights and thresholds based on real-time feedback from model performance or backtesting results.

    This analysis and enhancement plan aim to refine the functionality around institutional behavior detection, improving clarity, robustness, and adaptability in the market microstructure environment.

  9. User

    Between calculate_institutional_score1, calculate_institutional_score2 and calculate_institutional_score3, wic one is better and robust def calculate_institutional_score1(self, institutional_analysis: Dict[str, Any]) -> float: """Calculate the institutional score based on order characteristics and clustering.""" try: # Extract necessary information large_bids = institutional_analysis['large_orders']['bids'] large_asks = institutional_analysis['large_orders']['asks'] iceberg_prob = institutional_analysis.get('iceberg_probability', 0.0) bid_clusters = institutional_analysis['clustering']['bids'] ask_clusters = institutional_analysis['clustering']['asks']

        # Define weights for scoring
        large_order_weight = 0.3
        iceberg_weight = 0.3
        clustering_weight = 0.2
        market_impact_weight = 0.2
    
        # Calculate scores
        large_order_score = (len(large_bids) + len(large_asks)) / 10.0  # Example normalization
        clustering_score = (len(bid_clusters) + len(ask_clusters)) / 10.0
    
        # Example market impact calculation based on known impact factors
        market_impact_score = min(1.0, 0.1 * (len(large_bids) + len(large_asks)))
    
        # Total score computation
        institutional_score = (
            large_order_score * large_order_weight +
            iceberg_prob * iceberg_weight +
            clustering_score * clustering_weight +
            market_impact_score * market_impact_weight
        )
    
        # Normalize institutional_score to a range of 0 - 1
        return min(institutional_score, 1.0)
    
    except Exception as e:
        logger.error(f"Error calculating institutional score: {str(e)}")
        return 0.0
    
    def calculate_institutional_score2(self, institutional_analysis: Dict[str, Any]) -> float:
        """Calculate institutional score based on various factors."""
        try:
            large_orders_count = len(institutional_analysis['large_orders']['bids']) + len(institutional_analysis['large_orders']['asks'])
            iceberg_probability = institutional_analysis.get('iceberg_probability', 0.0)
            clustering_score = 0
    
            # Clustering analysis: Check for concentrated large orders
            for side in ['bids', 'asks']:
                cluster_volume = sum(institutional_analysis['clustering'][side].values())
                total_volume = sum(order['size'] for order in institutional_analysis['large_orders'][side])
                clustering_score += (cluster_volume / total_volume) if total_volume > 0 else 0
    
            clustering_score /= 2  # Average score from both sides
    
            # Smart routing suggestion influence
            smart_routing = institutional_analysis.get('smart_routing_suggestion', {})
            routing_influence = 0
            if smart_routing.get('execution_style') == 'careful':
                routing_influence = 0.2
            elif smart_routing.get('execution_style') == 'aggressive':
                routing_influence = -0.1
    
            # Combine factors with weights
            institutional_score = (
                large_orders_count * 0.3 +
                iceberg_probability * 0.4 +
                clustering_score * 0.2 +
                routing_influence
            )
    
            return max(min(institutional_score, 1.0), -1.0)  # Ensure score within -1 to 1
    
        except Exception as e:
            logger.error(f"Error calculating institutional score: {str(e)}")
            return 0.0
    

    def calculate_institutional_score3(self, institutional_analysis: Dict[str, Any]) -> float: """Calculate institutional score based on institutional_analysis data.""" try: # Sum up sizes of large orders total_large_order_volume = sum(order['size'] for order in institutional_analysis['large_orders']['bids'])
    + sum(order['size'] for order in institutional_analysis['large_orders']['asks'])

        # Sum up volumes in significant clusters
        total_cluster_volume = sum(institutional_analysis['clustering']['bids'].values()) \
                               + sum(institutional_analysis['clustering']['asks'].values())
        
        # Get iceberg probability
        iceberg_probability = institutional_analysis.get('iceberg_probability', 0.0)
        
        # Normalize volumes (assuming the institutional threshold is significant volume)
        normalized_large_order_volume = min(total_large_order_volume / (10 * self.institutional_threshold), 1.0)
    
        normalized_cluster_volume = min(total_cluster_volume / (10 * self.institutional_threshold), 1.0)
    
        # Now define weights
        weight_large_orders = 0.4
        weight_clusters = 0.4
        weight_iceberg = 0.2
    
        # Compute institutional score
        institutional_score = (normalized_large_order_volume * weight_large_orders +
                               normalized_cluster_volume * weight_clusters +
                               iceberg_probability * weight_iceberg)
        
        # Ensure the score is between 0 and 1
        institutional_score = max(min(institutional_score, 1.0), 0.0)
    
        return institutional_score
    
    except Exception as e:
        logger.error(f"Error calculating institutional score: {str(e)}")
        return 0.0
    
  10. Assistant

    To evaluate which version among calculate_institutional_score1, calculate_institutional_score2, and calculate_institutional_score3 is better and more robust, let's consider several factors:

    Key Evaluation Metrics

    1. Data Utilization: The extent to which each function uses available data within institutional_analysis effectively.
    2. Robustness: How well the function handles potential edge cases or anomalies in data.
    3. Weighting Logic: The appropriateness and rationale behind weights assigned to different factors contributing to the institutional score.
    4. Normalization: Ensuring scores are properly normalized and remain within reasonable bounds.
    5. Clear and Understandable Logic: Transparent and logical calculation processes which are easy to interpret.

    Analysis

    1. calculate_institutional_score1

    • Pros:

      • Uses a relatively detailed list of factors: large orders, iceberg probability, clustering, and an assumed market impact.
      • Introduces a straightforward and balanced weighting system.
    • Cons:

      • Market impact measure (market_impact_score) could be better defined and aligned with actual market conditions.
      • Normalization might not capture subtleties in very large or very small order volumes effectively.

    2. calculate_institutional_score2

    • Pros:

      • Includes a detailed look into smart routing suggestions, potentially indicating response strategies to market conditions.
      • Considers clustering as a calculated score based on volume concentration.
    • Cons:

      • The approach to handling smart routing (routing_influence) could lead to skewed weightings without strong behavioral indicators.
      • Provides a normalization range of -1 to 1, which suggests it could introduce negative factors; this could be unintended in certain calculation contexts.

    3. calculate_institutional_score3

    • Pros:

      • Focused use of both large order volume and cluster volume, incorporating iceberg probability systematically.
      • Effective normalization scheme using the institutional_threshold, providing clarity in volume significance.
      • Ensures the score adheres to a strict 0 to 1 range, preventing unrealistic negative scores.
    • Cons:

      • Simplified approach which largely depends on volumes and their direct normalization; some complexity might be warranted for nuanced market behaviors.
      • Potential for slightly skewed interpretation of clustering impact due to straightforward addition.

    Recommendation

    Best Option: calculate_institutional_score3

    • Reasoning: This version offers a precise use of the available data, focusing sharply on large orders, clustering, and iceberg probability. It manages to maintain robustness through clear normalization strategies, avoiding unintended negative scores which could misrepresent institutional presence. Moreover, the straightforward approach reduces the complexity inherent in potential misinterpretation, making it preferable for real-world applications where clear, consistent metrics are crucial.

    Enhancing calculate_institutional_score3 further with dynamic adjustments based on live market feedback could continue to build on its solid foundation, ensuring adaptive responsiveness in dynamic trading environments.

Responses were produced in different, uncontrolled contexts (system prompts, settings, prior turns). This is not a benchmark.