import asyncio import datetime import json import os import sys import threading import…
- User
import asyncio import datetime import json import os import sys import threading import time import django import pytz import websockets
Set the environment variable to point to your project’s settings
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'binancebot.settings')
Initialize Django
django.setup()
import concurrent.futures from asgiref.sync import sync_to_async from concurrent.futures import ProcessPoolExecutor, as_completed import ccxt import pandas as pd import numpy as np import talib as ta from django.utils import timezone from .models import Signal from multiprocessing import Process, Event from loguru import logger
Remove all existing handlers (including the default one)
logger.remove()
Configure your custom handlers
logger.add("trading_bot.log", rotation="20 MB", retention="2 days", level="DEBUG", format="{time:DD-MM-YYYY HH:mm:ss.SSS} | {level} | {message}") # File logging logger.add(sys.stdout, level="DEBUG") # Console logging
Ensure that the time displayed in logs is in Nairobi time
class SignalGenerator: def init(self, symbol='BTC/USDT', timeframe='5m'): self.symbol = symbol self.timeframe = timeframe.lower() # For perpetual futures on Binance Futures symbol_formatted = self.symbol.replace('/', '').lower() self.url = f"wss://fstream.binance.com/ws/{symbol_formatted}@kline_{self.timeframe}" ##self.url = f"wss://stream.binance.com:9443/ws/{self.symbol}@kline_{self.timeframe}" self.exchange = ccxt.binanceusdm({ 'enableRateLimit': True, }) self.exchange.load_markets() self.df = pd.DataFrame() self.data_lock = asyncio.Lock() logger.debug(f"Initialized SignalGenerator with symbol: {symbol}, timeframe: {timeframe}")
def fetch_historical_data(self): try: since = self.exchange.parse8601('2024-07-07T00:00:00Z') historical_bars = self.exchange.fetch_ohlcv( self.symbol, self.timeframe, since=since, ) df = pd.DataFrame(historical_bars, columns=['timestamp', 'open', 'high', 'low', 'close', 'volume']) df['timestamp'] = pd.to_datetime(df['timestamp'], unit='ms', utc=True) df['timestamp'] = df['timestamp'].dt.tz_convert('Africa/Nairobi') logger.info(f"Fetched historical data for {self.symbol} successfully.") return df except Exception as e: logger.error(f"Error fetching historical data for {self.symbol}: {e}") return pd.DataFrame() async def fetch_realtime_data(self): try: async with websockets.connect(self.url) as websocket: logger.info(f"Connected to websocket for {self.symbol}") while True: message = await websocket.recv() data = json.loads(message) # Extract kline data kline = data['k'] kline_data = { 'timestamp': pd.to_datetime(kline['t'], unit='ms', utc=True).tz_convert('Africa/Nairobi'), 'open': float(kline['o']), 'high': float(kline['h']), 'low': float(kline['l']), 'close': float(kline['c']), 'volume': float(kline['v']), } await self.collect_data(kline_data) except Exception as e: logger.error(f"Websocket connection error for {self.symbol}: {e}") await asyncio.sleep(5) # Wait before retrying #continue #return pd.DataFrame() async def collect_data(self, kline_data): async with self.data_lock: kline_data['timestamp'] = kline_data['timestamp'].tz_convert('Africa/Nairobi') self.df = pd.concat([self.df, pd.DataFrame([kline_data])], ignore_index=True) self.df.drop_duplicates(subset='timestamp', keep='last', inplace=True) self.df.sort_values(by='timestamp', inplace=True) # Keep only the recent data to limit memory usage self.df = self.df.tail(1000) # Adjust as needed async def combine_data(self): # Fetch historical data self.df = self.fetch_historical_data() # Start real-time data gathering asyncio.create_task(self.fetch_realtime_data()) # Now proceed to generate signals periodically while True: await self.generate_and_save_signals() await asyncio.sleep(1) # Adjust the sleep time as needed async def generate_and_save_signals(self): if not self.df.empty: last_signal_row = self.generate_signals(self.df) if last_signal_row is not None: await self.save_signals(last_signal_row) else: logger.error(f"Signal generation returned None for {self.symbol}.") else: logger.error("No data fetched, cannot proceed with signal generation.") def generate_signals(self, df): if df.empty: logger.error("No data present for signal generation.") return None try: # Ensure the DataFrame is sorted by timestamp df = df.sort_values(by='timestamp') # Use the last 50 rows for indicator calculations recent_df = df.tail(250).copy() # Check if there's enough data to compute indicators if len(recent_df) < 250: logger.warning("Not enough data to perform full technical analysis.") return None # Calculate indicators on recent data # Calculate EMAs recent_df['EMA20'] = ta.EMA(recent_df['close'], timeperiod=20) recent_df['EMA50'] = ta.EMA(recent_df['close'], timeperiod=50) # Calculate Ichimoku Cloud components without future shifts # Tenkan-sen (Conversion Line) nine_period_high = recent_df['high'].rolling(window=9).max() nine_period_low = recent_df['low'].rolling(window=9).min() recent_df['tenkan_sen'] = (nine_period_high + nine_period_low) / 2 # Kijun-sen (Base Line) period26_high = recent_df['high'].rolling(window=26).max() period26_low = recent_df['low'].rolling(window=26).min() recent_df['kijun_sen'] = (period26_high + period26_low) / 2 # Senkou Span A (Leading Span A) recent_df['senkou_span_a'] = ((recent_df['tenkan_sen'] + recent_df['kijun_sen']) / 2) # Senkou Span B (Leading Span B) period52_high = recent_df['high'].rolling(window=52).max() period52_low = recent_df['low'].rolling(window=52).min() recent_df['senkou_span_b'] = (period52_high + period52_low) / 2 # Chikou Span (Lagging Span) # Shifted backward by 26 periods recent_df['chikou_span'] = recent_df['close'].shift(-26) # Drop NaN values resulting from calculations recent_df.dropna(inplace=True) # Calculate ATR recent_df['ATR'] = ta.ATR(recent_df['high'], recent_df['low'], recent_df['close'], timeperiod=14) # Calculate RSI recent_df['RSI'] = ta.RSI(recent_df['close'], timeperiod=14) # Focus on the last data point for signal generation last_row = recent_df.iloc[-1] previous_row = recent_df.iloc[-2] # For comparing previous indicator values # Generate signals based on conditions bullish_cross = ( (last_row['EMA20'] > last_row['EMA50']) and (previous_row['EMA20'] <= previous_row['EMA50']) ) bearish_cross = ( (last_row['EMA20'] < last_row['EMA50']) and (previous_row['EMA20'] >= previous_row['EMA50']) ) # Price above/below Ichimoku Cloud senkou_span_a = last_row['senkou_span_a'] senkou_span_b = last_row['senkou_span_b'] # Handle potential NaN values due to shifting if pd.isna(senkou_span_a) or pd.isna(senkou_span_b): logger.warning("Senkou spans contain NaN values after shifting.") return None max_senkou_span = max(senkou_span_a, senkou_span_b) min_senkou_span = min(senkou_span_a, senkou_span_b) price_above_cloud = last_row['close'] > max_senkou_span price_below_cloud = last_row['close'] < min_senkou_span # RSI conditions rsi_overbought = last_row['RSI'] > 70 rsi_oversold = last_row['RSI'] < 30 # Check for NaN values in indicators indicators = ['EMA20', 'EMA50', 'tenkan_sen', 'kijun_sen', 'senkou_span_a', 'senkou_span_b', 'RSI'] for indicator in indicators: if pd.isna(last_row[indicator]) or pd.isna(previous_row[indicator]): logger.warning(f"{indicator} contains NaN values for {self.symbol}.") # Log indicator values logger.debug(f"{self.symbol} Indicators at {last_row['timestamp']}: EMA20={last_row['EMA20']}, EMA50={last_row['EMA50']}, RSI={last_row['RSI']}") logger.debug(f"{self.symbol} Ichimoku at {last_row['timestamp']}: Senkou Span A={last_row['senkou_span_a']}, Senkou Span B={last_row['senkou_span_b']}") # Log condition values logger.debug(f"{self.symbol} Conditions at {last_row['timestamp']}: bullish_cross={bullish_cross}, bearish_cross={bearish_cross}, price_above_cloud={price_above_cloud}, price_below_cloud={price_below_cloud}, rsi_overbought={rsi_overbought}, rsi_oversold={rsi_oversold}") # Initialize signal #signal = 'HOLD' #if used remove else below # Determine signal based on conditions if bullish_cross and price_above_cloud and not rsi_overbought: signal = 'BUY' elif bearish_cross and price_below_cloud and not rsi_oversold: signal = 'SELL' else: signal = 'HOLD' #if used remove initialized above # Assign the signal to the last row last_row['Signal'] = signal # Log the generated signal logger.info(f"Generated signal for {self.symbol} at {last_row['timestamp']}: {signal}") # Return the last row as a DataFrame to keep consistency return last_row.to_frame().T except Exception as e: logger.error(f"Error generating signals for {self.symbol}: {e}") return None async def save_signals(self, last_signal_row): try: last_signal = last_signal_row.iloc[0] timestamp = last_signal['timestamp'] # Should be timezone-aware in Nairobi time await sync_to_async(Signal.objects.update_or_create)( timestamp=timestamp, symbol=self.symbol, defaults={ 'price': last_signal['close'], 'signal_type': last_signal['Signal'], } ) logger.info(f"Signal saved for {self.symbol} at {timestamp}: {last_signal['Signal']}") except Exception as e: logger.error(f"Error saving signals for {self.symbol} to database: {e}") async def run(self): try: await self.combine_data() if not self.df.empty: self.df = self.generate_signals(self.df) await self.save_signals(self.df) else: logger.error("No data fetched, cannot proceed with signal generation.") #stop_event.wait(1) # Check the event every second except asyncio.CancelledError: logger.info(f"Task for {self.symbol} has been cancelled.") # Clean up resources if necessary raiseasync def main(): exchange = ccxt.binanceusdm({'enableRateLimit': True}) exchange.load_markets() markets = exchange.markets
Filter for USDT pairs
try: usdt_pairs = set() for symbol, market in markets.items(): if (market['symbol'].endswith(':USDT') and market.get('contract', False) and market.get('linear', False) and market.get('expiry') is None): # Normalize the symbol by removing ':USDT' normalized_symbol = market['symbol'].replace(':USDT', '') usdt_pairs.add(normalized_symbol) logger.info(f"Total Futures USDT Pairs: {len(usdt_pairs)}") if not usdt_pairs: logger.warning("No USDT pairs found after filtering. Please check the filtering criteria.") tasks = [] for symbol in usdt_pairs: logger.info(f"Processing symbol: {symbol}") signal_generator = SignalGenerator(symbol=symbol) tasks.append(asyncio.create_task(signal_generator.run())) await asyncio.gather(*tasks) except KeyboardInterrupt: logger.info("KeyboardInterrupt received. Shutting down executor…") # Setup tasks for real-time data retrievalif name == "main": asyncio.run(main()) In the above code, I want you to try and reduce the number of API calls and also limit the number of Concurrency to 6. The bot is supposed to continue running, however, I dont want to limit it to only work with 6 symbols only. I want it to process all symbols to catch any opportunities that may arise from other symbols. Just make necessary changes only, don't change the functionality or processes of the existing functions unnecessarily
- Assistant
Here's the modified code with the required changes:
import asyncio import datetime import json import os import sys import threading import time import django import pytz import websockets # Set the environment variable to point to your project’s settings os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'binancebot.settings') # Initialize Django django.setup() import concurrent.futures from asgiref.sync import sync_to_async from concurrent.futures import ProcessPoolExecutor, as_completed import ccxt import pandas as pd import numpy as np import talib as ta from django.utils import timezone from .models import Signal from multiprocessing import Process, Event from loguru import logger # Remove all existing handlers (including the default one) logger.remove() # Configure your custom handlers logger.add("trading_bot.log", rotation="20 MB", retention="2 days", level="DEBUG", format="{time:DD-MM-YYYY HH:mm:ss.SSS} | {level} | {message}") # File logging logger.add(sys.stdout, level="DEBUG") # Console logging # Ensure that the time displayed in logs is in Nairobi time class SignalGenerator: def __init__(self, symbols, timeframe='5m'): self.symbols = symbols # List of symbols self.timeframe = timeframe.lower() self.exchange = ccxt.binanceusdm({ 'enableRateLimit': True, }) self.exchange.load_markets() self.dfs = {symbol: pd.DataFrame() for symbol in self.symbols} self.data_lock = asyncio.Lock() logger.debug(f"Initialized SignalGenerator with symbols: {symbols}, timeframe: {timeframe}") def fetch_historical_data(self): dfs = {} for symbol in self.symbols: try: limit = 500 # Fetch recent 500 data points historical_bars = self.exchange.fetch_ohlcv( symbol, self.timeframe, limit=limit, ) df = pd.DataFrame(historical_bars, columns=['timestamp', 'open', 'high', 'low', 'close', 'volume']) df['timestamp'] = pd.to_datetime(df['timestamp'], unit='ms', utc=True) df['timestamp'] = df['timestamp'].dt.tz_convert('Africa/Nairobi') logger.info(f"Fetched historical data for {symbol} successfully.") dfs[symbol] = df except Exception as e: logger.error(f"Error fetching historical data for {symbol}: {e}") dfs[symbol] = pd.DataFrame() return dfs async def fetch_realtime_data(self): ws_url = 'wss://fstream.binance.com/ws' try: async with websockets.connect(ws_url) as websocket: logger.info(f"Connected to websocket for symbols: {self.symbols}") # Send subscription message subscribe_params = [symbol.replace('/', '').lower() + f'@kline_{self.timeframe}' for symbol in self.symbols] subscribe_message = { "method": "SUBSCRIBE", "params": subscribe_params, "id": 1 } await websocket.send(json.dumps(subscribe_message)) while True: message = await websocket.recv() data = json.loads(message) if 'result' in data: continue # Skip subscription confirmation messages stream = data['stream'] symbol_stream = stream.split('@')[0].upper() # e.g., 'BTCUSDT' symbol = f"{symbol_stream[:-4]}/{symbol_stream[-4:]}" kline = data['data']['k'] kline_data = { 'timestamp': pd.to_datetime(kline['t'], unit='ms', utc=True).tz_convert('Africa/Nairobi'), 'open': float(kline['o']), 'high': float(kline['h']), 'low': float(kline['l']), 'close': float(kline['c']), 'volume': float(kline['v']), } await self.collect_data(symbol, kline_data) except Exception as e: logger.error(f"Websocket connection error for symbols {self.symbols}: {e}") await asyncio.sleep(5) # Wait before retrying async def collect_data(self, symbol, kline_data): async with self.data_lock: kline_data['timestamp'] = kline_data['timestamp'].tz_convert('Africa/Nairobi') self.dfs[symbol] = pd.concat([self.dfs[symbol], pd.DataFrame([kline_data])], ignore_index=True) self.dfs[symbol].drop_duplicates(subset='timestamp', keep='last', inplace=True) self.dfs[symbol].sort_values(by='timestamp', inplace=True) # Keep only the recent data to limit memory usage self.dfs[symbol] = self.dfs[symbol].tail(1000) # Adjust as needed async def combine_data(self): # Fetch historical data self.dfs = self.fetch_historical_data() # Start real-time data gathering asyncio.create_task(self.fetch_realtime_data()) # Now proceed to generate signals periodically while True: await self.generate_and_save_signals() await asyncio.sleep(1) # Adjust the sleep time as needed async def generate_and_save_signals(self): for symbol, df in self.dfs.items(): if not df.empty: last_signal_row = self.generate_signals(df, symbol) if last_signal_row is not None: await self.save_signals(last_signal_row, symbol) else: logger.error(f"Signal generation returned None for {symbol}.") else: logger.error(f"No data fetched for {symbol}, cannot proceed with signal generation.") def generate_signals(self, df, symbol): if df.empty: logger.error(f"No data present for signal generation for {symbol}.") return None try: # Ensure the DataFrame is sorted by timestamp df = df.sort_values(by='timestamp') # Use the last 250 rows for indicator calculations recent_df = df.tail(250).copy() # Check if there's enough data to compute indicators if len(recent_df) < 250: logger.warning(f"Not enough data to perform full technical analysis for {symbol}.") return None # Calculate indicators on recent data # Calculate EMAs recent_df['EMA20'] = ta.EMA(recent_df['close'], timeperiod=20) recent_df['EMA50'] = ta.EMA(recent_df['close'], timeperiod=50) # Calculate Ichimoku Cloud components without future shifts # Tenkan-sen (Conversion Line) nine_period_high = recent_df['high'].rolling(window=9).max() nine_period_low = recent_df['low'].rolling(window=9).min() recent_df['tenkan_sen'] = (nine_period_high + nine_period_low) / 2 # Kijun-sen (Base Line) period26_high = recent_df['high'].rolling(window=26).max() period26_low = recent_df['low'].rolling(window=26).min() recent_df['kijun_sen'] = (period26_high + period26_low) / 2 # Senkou Span A (Leading Span A) recent_df['senkou_span_a'] = ((recent_df['tenkan_sen'] + recent_df['kijun_sen']) / 2) # Senkou Span B (Leading Span B) period52_high = recent_df['high'].rolling(window=52).max() period52_low = recent_df['low'].rolling(window=52).min() recent_df['senkou_span_b'] = (period52_high + period52_low) / 2 # Chikou Span (Lagging Span) # Shifted backward by 26 periods recent_df['chikou_span'] = recent_df['close'].shift(-26) # Drop NaN values resulting from calculations recent_df.dropna(inplace=True) # Calculate ATR recent_df['ATR'] = ta.ATR(recent_df['high'], recent_df['low'], recent_df['close'], timeperiod=14) # Calculate RSI recent_df['RSI'] = ta.RSI(recent_df['close'], timeperiod=14) # Focus on the last data point for signal generation last_row = recent_df.iloc[-1] previous_row = recent_df.iloc[-2] # For comparing previous indicator values # Generate signals based on conditions bullish_cross = ( (last_row['EMA20'] > last_row['EMA50']) and (previous_row['EMA20'] <= previous_row['EMA50']) ) bearish_cross = ( (last_row['EMA20'] < last_row['EMA50']) and (previous_row['EMA20'] >= previous_row['EMA50']) ) # Price above/below Ichimoku Cloud senkou_span_a = last_row['senkou_span_a'] senkou_span_b = last_row['senkou_span_b'] # Handle potential NaN values due to shifting if pd.isna(senkou_span_a) or pd.isna(senkou_span_b): logger.warning(f"Senkou spans contain NaN values after shifting for {symbol}.") return None max_senkou_span = max(senkou_span_a, senkou_span_b) min_senkou_span = min(senkou_span_a, senkou_span_b) price_above_cloud = last_row['close'] > max_senkou_span price_below_cloud = last_row['close'] < min_senkou_span # RSI conditions rsi_overbought = last_row['RSI'] > 70 rsi_oversold = last_row['RSI'] < 30 # Check for NaN values in indicators indicators = ['EMA20', 'EMA50', 'tenkan_sen', 'kijun_sen', 'senkou_span_a', 'senkou_span_b', 'RSI'] for indicator in indicators: if pd.isna(last_row[indicator]) or pd.isna(previous_row[indicator]): logger.warning(f"{indicator} contains NaN values for {symbol}.") # Log indicator values logger.debug(f"{symbol} Indicators at {last_row['timestamp']}: EMA20={last_row['EMA20']}, EMA50={last_row['EMA50']}, RSI={last_row['RSI']}") logger.debug(f"{symbol} Ichimoku at {last_row['timestamp']}: Senkou Span A={last_row['senkou_span_a']}, Senkou Span B={last_row['senkou_span_b']}") # Log condition values logger.debug(f"{symbol} Conditions at {last_row['timestamp']}: bullish_cross={bullish_cross}, bearish_cross={bearish_cross}, price_above_cloud={price_above_cloud}, price_below_cloud={price_below_cloud}, rsi_overbought={rsi_overbought}, rsi_oversold={rsi_oversold}") # Determine signal based on conditions if bullish_cross and price_above_cloud and not rsi_overbought: signal = 'BUY' elif bearish_cross and price_below_cloud and not rsi_oversold: signal = 'SELL' else: signal = 'HOLD' #if used remove initialized above # Assign the signal to the last row last_row['Signal'] = signal # Log the generated signal logger.info(f"Generated signal for {symbol} at {last_row['timestamp']}: {signal}") # Return the last row as a DataFrame to keep consistency return last_row.to_frame().T except Exception as e: logger.error(f"Error generating signals for {symbol}: {e}") return None async def save_signals(self, last_signal_row, symbol): try: last_signal = last_signal_row.iloc[0] timestamp = last_signal['timestamp'] # Should be timezone-aware in Nairobi time await sync_to_async(Signal.objects.update_or_create)( timestamp=timestamp, symbol=symbol, defaults={ 'price': last_signal['close'], 'signal_type': last_signal['Signal'], } ) logger.info(f"Signal saved for {symbol} at {timestamp}: {last_signal['Signal']}") except Exception as e: logger.error(f"Error saving signals for {symbol} to database: {e}") async def run(self): try: await self.combine_data() except asyncio.CancelledError: logger.info(f"Task for symbols {self.symbols} has been cancelled.") # Clean up resources if necessary raise async def main(): exchange = ccxt.binanceusdm({'enableRateLimit': True}) exchange.load_markets() markets = exchange.markets # Filter for USDT pairs try: usdt_pairs = set() for symbol, market in markets.items(): if (market['symbol'].endswith(':USDT') and market.get('contract', False) and market.get('linear', False) and market.get('expiry') is None): # Normalize the symbol by removing ':USDT' normalized_symbol = market['symbol'].replace(':USDT', '') usdt_pairs.add(normalized_symbol) logger.info(f"Total Futures USDT Pairs: {len(usdt_pairs)}") if not usdt_pairs: logger.warning("No USDT pairs found after filtering. Please check the filtering criteria.") tasks = [] symbols_list = list(usdt_pairs) # Limit concurrency to 6 num_batches = min(6, len(symbols_list)) batch_size = (len(symbols_list) + num_batches - 1) // num_batches # Ceiling division symbol_batches = [symbols_list[i:i + batch_size] for i in range(0, len(symbols_list), batch_size)] for batch_symbols in symbol_batches: logger.info(f"Processing batch symbols: {batch_symbols}") signal_generator = SignalGenerator(symbols=batch_symbols) tasks.append(asyncio.create_task(signal_generator.run())) await asyncio.gather(*tasks) except KeyboardInterrupt: logger.info("KeyboardInterrupt received. Shutting down executor…") # Setup tasks for real-time data retrieval if __name__ == "__main__": asyncio.run(main())Explanation of Changes:
-
Limited Concurrency to 6:
- In the
main()function, we divide the symbols into batches with a maximum of 6 batches. This ensures that at any given time, only 6SignalGeneratorinstances are running concurrently. - Each
SignalGeneratorhandles a batch of symbols.
- In the
-
Reduced Number of WebSocket Connections:
- Instead of creating a WebSocket connection for each symbol, we modified the
fetch_realtime_data()method to subscribe to multiple symbols within a single WebSocket connection using the Binance subscription model. - This reduces the number of WebSocket connections significantly.
- Instead of creating a WebSocket connection for each symbol, we modified the
-
Optimized API Calls for Historical Data:
- In the
fetch_historical_data()method, we limited the number of data points fetched per symbol by setting alimitparameter (e.g., 500). - This reduces the number of API calls and the amount of data fetched, helping to stay within rate limits.
- In the
-
Adjusted Functions to Handle Multiple Symbols:
- Modified the
SignalGeneratorclass to accept a list of symbols (symbols) instead of a single symbol. - Updated methods like
fetch_historical_data(),fetch_realtime_data(),collect_data(),generate_and_save_signals(),generate_signals(), andsave_signals()to handle multiple symbols. - DataFrames are stored in a dictionary
self.dfswith symbols as keys.
- Modified the
-
Using Binance Subscription Model:
- Switched to sending a subscription message to the WebSocket server after connecting, avoiding long URLs and potential connection issues.
- Handled incoming messages by parsing the
streamfield to determine which symbol the data belongs to.
-
Preserved Existing Functionality:
- The core functionality and processes within existing functions remain unchanged.
- Adjustments were made only where necessary to accommodate the handling of multiple symbols and to limit concurrency.
Note: This code assumes that your environment has the necessary infrastructure to handle multiple symbols and that your database and logging systems are properly configured to accommodate the changes.
-
Other models, same or similar prompt
Responses were produced in different, uncontrolled contexts (system prompts, settings, prior turns). This is not a benchmark.