please add # In advancedtrading.py class AdvancedTrading: def init(self): self.metrics =…
- User
please add # In advanced_trading.py class AdvancedTrading: def init(self): self._metrics = { 'trades_executed': 0, 'total_volume': 0.0, 'successful_trades': 0, 'failed_trades': 0, 'average_latency': 0.0, 'peak_latency': 0.0, 'profit_loss': 0.0 }
# Add process queue self.process_queue = asyncio.Queue(maxsize=int(os.getenv('QUEUE_SIZE', 1000))) def _save_metrics(self): """Save trading metrics""" try: metrics_data = { **self._metrics, 'timestamp': time.time(), 'session_duration': time.time() - self._start_time } logger.info(f"Trading Metrics: {json.dumps(metrics_data, indent=2)}") except Exception as e: logger.error(f"Error saving metrics: {e}") async def cleanup(self): try: self._save_metrics() # Clear process queue while not self.process_queue.empty(): try: self.process_queue.get_nowait() except asyncio.QueueEmpty: break logger.info("Advanced trading cleanup completed") except Exception as e: logger.error(f"Advanced trading cleanup error: {e}") to ----> this >>>> import asyncioimport base64 import json import logging import os import time from typing import Dict, List, Optional from decimal import Decimal import torch import torch.cuda.amp as amp from dataclasses import dataclass, field import psutil import websockets from concurrent.futures import ThreadPoolExecutor from dotenv import load_dotenv from config_loader import config
Load environment variables from .env file
load_dotenv()
logging.basicConfig(level=logging.DEBUG, format='%(asctime)s - %(levelname)s - %(message)s') logger = logging.getLogger(name)
@dataclass class TradingMetrics: messages_received: int = 0 opportunities_found: int = 0 successful_trades: int = 0 total_profit: Decimal = Decimal('0') start_time: float = field(default_factory=time.time) sandwich_profits: Decimal = Decimal('0') impact_profits: Decimal = Decimal('0') failed_attempts: Dict[str, int] = field(default_factory=dict)
strategy_performance: Dict[str, Dict] = field(default_factory=lambda: { 'sandwich': {'success': 0, 'fail': 0, 'avg_profit': Decimal('0'), 'avg_time': 0.0}, 'impact': {'success': 0, 'fail': 0, 'avg_profit': Decimal('0'), 'avg_time': 0.0} }) execution_stats: Dict[str, List[float]] = field(default_factory=lambda: { 'latency': [], 'gas_used': [], 'slippage': [] })class AdvancedTrading: def init(self, trader, config): self.trader = trader self.config = config self.active = False self.metrics = TradingMetrics()
# GPU setup with tensor cores enabled self.device = torch.device('cuda') torch.backends.cuda.matmul.allow_tf32 = True torch.backends.cudnn.benchmark = True # Initialize CUDA streams with priorities self.streams = { 'sandwich': torch.cuda.Stream(priority=-2), # Highest priority 'impact': torch.cuda.Stream(priority=-1), # High priority 'compute': torch.cuda.Stream(priority=0) # Normal priority } # Initialize thread pool for CPU tasks cpu_count = psutil.cpu_count(logical=False) self.thread_pool = ThreadPoolExecutor(max_workers=cpu_count) # Setup processing buffers self.setup_processing_buffers() # Initialize opportunity queue with prioritization self.opportunity_queue = asyncio.PriorityQueue() # Load trading parameters from environment variables self._load_trading_parameters() logger.info("Advanced trading initialized with GPU acceleration") def _load_trading_parameters(self): """Load and validate trading parameters""" try: # Load from environment variables self.MIN_PROFIT_PCT = float(os.getenv("MIN_PROFIT_PCT")) self.MAX_PROFIT_PCT = float(os.getenv("MAX_PROFIT_PCT")) self.MAX_PRICE_IMPACT = float(os.getenv("MAX_PRICE_IMPACT")) self.MAX_POSITION_SIZE = float(os.getenv("POSITION_MAX_SIZE")) self.INITIAL_SIZE = float(os.getenv("POSITION_INITIAL_SIZE")) self.GAS_RESERVE = float(os.getenv("GAS_RESERVE")) self.SLIPPAGE = float(os.getenv("MAX_SPREAD")) # Sandwich specific parameters self.MIN_SANDWICH_SIZE = float(os.getenv("SANDWICH_MIN_VICTIM_SIZE")) # Minimum victim tx size in SOL self.MAX_SANDWICH_IMPACT = float(os.getenv("SANDWICH_MAX_IMPACT")) # Maximum impact for sandwich trades self.SANDWICH_PROFIT_THRESHOLD = float(os.getenv("SANDWICH_MIN_PROFIT")) # Minimum profit threshold logger.info("Trading parameters loaded successfully") except KeyError as e: logger.error(f"Missing configuration parameter: {e}") raise def setup_processing_buffers(self): """Initialize optimized GPU processing buffers""" try: gpu_mem = torch.cuda.get_device_properties(0).total_memory buffer_size = min(2048, gpu_mem // (1024 * 1024 * 10)) # Create CPU tensors first self.opportunity_buffer = torch.zeros( (buffer_size, 128), dtype=torch.float16, device='cpu' # Create on CPU first ).pin_memory() # Pin the memory self.impact_buffer = torch.zeros( (buffer_size // 2, 2), dtype=torch.float32, device='cpu' ).pin_memory() self.pattern_buffer = torch.zeros( (1024, 64), dtype=torch.float16, device='cpu' ).pin_memory() # Move to GPU if available if torch.cuda.is_available(): with torch.cuda.stream(self.streams['compute']): self.opportunity_buffer = self.opportunity_buffer.to( device=self.device, non_blocking=True ) self.impact_buffer = self.impact_buffer.to( device=self.device, non_blocking=True ) self.pattern_buffer = self.pattern_buffer.to( device=self.device, non_blocking=True ) torch.cuda.synchronize() logger.debug("GPU buffers initialized") except Exception as e: logger.error(f"Failed to initialize GPU buffers: {e}") raise async def _check_sandwich_opportunity(self, raw_data: bytes) -> Optional[Dict]: """Check for sandwich opportunities with enhanced MEV protection""" try: with torch.cuda.stream(self.streams['sandwich']): # Decode and validate victim transaction victim_tx = await self._decode_victim_transaction(raw_data) if not victim_tx or victim_tx['amount'] < self.MIN_SANDWICH_SIZE: return None # Calculate optimal positions positions = await self._calculate_optimal_positions( victim_tx['pool'], victim_tx['amount'], victim_tx['direction'] ) if not positions: return None # MEV protection check is_safe = await self.trader.mev_protection.verify_sandwich( positions['front_run'], victim_tx['amount'], positions['back_run'] ) if not is_safe: return None # Calculate expected profit profit = await self._calculate_sandwich_profit(positions, victim_tx['pool']) if profit < self.SANDWICH_PROFIT_THRESHOLD: return None return { 'type': 'sandwich', 'profit': profit, 'positions': positions, 'victim': victim_tx, 'pool': victim_tx['pool'], 'confidence': 0.9, 'priority': 1 } except Exception as e: logger.error(f"Sandwich check error: {e}") return None async def _calculate_optimal_positions(self, pool_address: str, victim_amount: float, direction: bool) -> Optional[Dict]: """Calculate optimal front-run and back-run positions""" try: with torch.cuda.stream(self.streams['sandwich']): # Get pool state pool_state = await self.trader.get_pool_state(pool_address) if not pool_state: return None # Convert to tensors for GPU calculation base_reserve = torch.tensor(float(pool_state.base_reserve), device=self.device) quote_reserve = torch.tensor(float(pool_state.quote_reserve), device=self.device) victim_amount_tensor = torch.tensor(victim_amount, device=self.device) # Calculate optimal front-run size with amp.autocast(device_type='cuda'): # Start with 50% of victim tx size front_run_size = victim_amount_tensor * 0.5 # Iteratively optimize front-run size for _ in range(5): # Max 5 iterations # Calculate price impact if direction: # Victim buying new_quote_reserve = quote_reserve + front_run_size new_base_reserve = base_reserve * quote_reserve / new_quote_reserve price_impact = (new_base_reserve / base_reserve - 1).abs() else: # Victim selling new_base_reserve = base_reserve + front_run_size new_quote_reserve = quote_reserve * base_reserve / new_base_reserve price_impact = (new_quote_reserve / quote_reserve - 1).abs() # Adjust size based on impact if price_impact > self.MAX_PRICE_IMPACT: front_run_size *= 0.9 else: break # Calculate back-run size back_run_size = front_run_size * 1.1 # Slightly larger to account for slippage # Calculate minimum output amounts with slippage buffer front_run_min_out = self._calculate_min_output( float(front_run_size), float(base_reserve), float(quote_reserve), self.SLIPPAGE ) back_run_min_out = self._calculate_min_output( float(back_run_size), float(new_base_reserve), float(new_quote_reserve), self.SLIPPAGE ) return { 'front_run': float(front_run_size), 'back_run': float(back_run_size), 'front_run_min_out': front_run_min_out, 'back_run_min_out': back_run_min_out, 'price_impact': float(price_impact) } except Exception as e: logger.error(f"Position calculation error: {e}") return None async def _execute_sandwich_trade(self, opportunity: Dict, priority_fee: int) -> bool: """Execute sandwich trade with precise timing""" try: # Execute front-run with higher priority front_run_fee = int(priority_fee * 1.2) # 20% higher for front-run front_run_result = await self.trader.trading_core.execute_trade( pool_address=opportunity['pool'], amount_in=opportunity['positions']['front_run'], min_amount_out=opportunity['positions']['front_run_min_out'], priority_fee=front_run_fee, compute_limit=600000 # Higher compute limit for complex trades ) if not front_run_result: logger.error("Front-run execution failed") return False # Minimal waiting period for victim transaction await asyncio.sleep(0.05) # 50ms delay # Back-run with lower priority to save on fees back_run_fee = int(priority_fee * 0.8) # 20% lower for back-run back_run_result = await self.trader.trading_core.execute_trade( pool_address=opportunity['pool'], amount_in=opportunity['positions']['back_run'], min_amount_out=opportunity['positions']['back_run_min_out'], priority_fee=back_run_fee ) if not back_run_result: logger.error("Back-run execution failed") return False return True except Exception as e: logger.error(f"Sandwich execution error: {e}") return False def _calculate_min_output(self, amount_in: float, base_reserve: float, quote_reserve: float, slippage: float) -> float: """Calculate minimum output amount with slippage protection""" try: constant_product = base_reserve * quote_reserve new_quote_reserve = quote_reserve + amount_in ideal_output = base_reserve - (constant_product / new_quote_reserve) min_output = ideal_output * (1 - slippage) return max(min_output, 0) except Exception as e: logger.error(f"Min output calculation error: {e}") return 0 async def _optimize_priority_fee(self, opportunity_type: str, profit: float) -> int: """Dynamic priority fee optimization""" try: base_fee = await self.trader.mev_protection.get_priority_fee() multipliers = { 'sandwich': 1.5, # Higher priority for sandwich 'impact': 1.2 # Medium priority for impact trades } fee_multiplier = multipliers.get(opportunity_type, 1.0) if profit > float(os.getenv("PROFIT_THRESHOLD")): fee_multiplier *= float(os.getenv("FEE_MULTIPLIER")) priority_fee = int(base_fee * fee_multiplier) return min( max(priority_fee, int(os.getenv("MIN_PRIORITY_FEE"))), int(os.getenv("MAX_PRIORITY_FEE")) ) except Exception as e: logger.error(f"Fee optimization error: {e}") return int(os.getenv("DEFAULT_PRIORITY_FEE")) async def execute_opportunity(self, opportunity: Dict) -> bool: """Execute opportunity with dynamic fee optimization""" start_time = time.time() try: if not await self._validate_execution_conditions(opportunity): return False priority_fee = await self._optimize_priority_fee( opportunity['type'], opportunity['profit'] ) if opportunity['type'] == 'sandwich': result = await self._execute_sandwich_trade(opportunity, priority_fee) else: # impact trade result = await self._execute_impact_trade(opportunity, priority_fee) if result: execution_time = time.time() - start_time await self._update_execution_stats(opportunity, execution_time) self.metrics.successful_trades += 1 if opportunity['type'] == 'sandwich': self.metrics.sandwich_profits += Decimal(str(opportunity['profit'])) else: self.metrics.impact_profits += Decimal(str(opportunity['profit'])) logger.info(f"Trade successful: {opportunity['type']}, Profit: {opportunity['profit']:.4f}") return True return False except Exception as e: logger.error(f"Execution error: {e}") execution_time = time.time() - start_time await self._update_failed_execution_stats(opportunity, execution_time) return False async def start_websocket_monitoring(self): """High-performance WebSocket monitoring""" self.active = True logger.info("Starting advanced trading monitor...") while self.active: try: async with websockets.connect( os.getenv("HELIUS_WS_URL"), ping_interval=int(os.getenv("WS_PING_INTERVAL")), ping_timeout=int(os.getenv("WS_PING_TIMEOUT")), close_timeout=int(os.getenv("WS_CLOSE_TIMEOUT")), max_size=int(os.getenv("WS_MAX_MESSAGE_SIZE")) ) as ws: await ws.send(json.dumps({ "jsonrpc": "2.0", "id": 1, "method": "logsSubscribe", "params": [ {"mentions": [self.config['programs']['raydium']]}, {"commitment": "processed"} ] })) processors = [ self._process_messages(ws), self._monitor_network_conditions(), self._optimize_execution_parameters() ] await asyncio.gather(*processors) except websockets.exceptions.ConnectionClosed: logger.warning("WebSocket connection closed. Reconnecting...") await asyncio.sleep(0.1) except Exception as e: logger.error(f"WebSocket error: {e}") await asyncio.sleep(0.1) async def _process_messages(self, ws): """Process messages with GPU acceleration""" while self.active: try: msg = await ws.recv() self.metrics.messages_received += 1 with amp.autocast(device_type='cuda'): opportunities = await self._parallel_opportunity_check(msg) if opportunities: await self._execute_best_opportunity(opportunities) except Exception as e: logger.error(f"Message processing error: {e}") @torch.amp.autocast('cuda') async def _parallel_opportunity_check(self, msg: str) -> List[Dict]: """Check for sandwich and price impact opportunities in parallel""" try: data = json.loads(msg) if 'params' not in data: return [] logs = data['params'].get('result', {}).get('value', {}).get('logs', []) opportunities = [] for log in logs: if "ray_log:" not in log: continue raw_data = base64.b64decode(log.split('ray_log: ')[1].strip()) # Parallel opportunity checking with torch.cuda.stream(self.streams['compute']): results = await asyncio.gather( self._check_sandwich_opportunity(raw_data), self._check_price_impact_opportunity(raw_data) ) valid_results = [r for r in results if r and self._validate_opportunity(r)] if valid_results: opportunities.extend(self._calculate_opportunity_scores(valid_results)) return opportunities except Exception as e: logger.error(f"Parallel check error: {e}") return [] async def _check_price_impact_opportunity(self, raw_data: bytes) -> Optional[Dict]: """Check for price impact opportunities""" try: with torch.cuda.stream(self.streams['impact']): # Extract transaction data tx_data = await self._process_trade_data(raw_data) if not tx_data: return None # Calculate impact and profit impact = await self._calculate_price_impact( tx_data['amount_in'], tx_data['amount_out'], tx_data['pool'] ) if impact > self.MAX_PRICE_IMPACT: return None profit = await self._calculate_impact_profit(tx_data, impact) if profit < self.MIN_PROFIT_PCT: return None return { 'type': 'impact', 'profit': profit, 'amount': tx_data['amount_in'], 'pool': tx_data['pool'], 'impact': impact, 'confidence': 0.85, 'priority': 2 } except Exception as e: logger.error(f"Impact check error: {e}") return None async def cleanup(self): """Cleanup resources properly""" logger.info("Starting cleanup...") try: self.active = False # Clean GPU resources torch.cuda.empty_cache() for stream in self.streams.values(): stream.synchronize() # Save performance metrics self._save_metrics() # Close thread pool self.thread_pool.shutdown(wait=True) logger.info("Cleanup completed") except Exception as e: logger.error(f"Cleanup error: {e}") raise - Assistant
Certainly! Below is the modified version of your
advanced_trading.pyfile with the addition of the newAdvancedTradingclass logic that saves metrics and cleans up resources. The existing methods and structures have been preserved as originally implemented.import asyncio import base64 import json import logging import os import time from typing import Dict, List, Optional from decimal import Decimal import torch import torch.cuda.amp as amp from dataclasses import dataclass, field import psutil import websockets from concurrent.futures import ThreadPoolExecutor from dotenv import load_dotenv from config_loader import config # Load environment variables from .env file load_dotenv() logging.basicConfig(level=logging.DEBUG, format='%(asctime)s - %(levelname)s - %(message)s') logger = logging.getLogger(__name__) @dataclass class TradingMetrics: messages_received: int = 0 opportunities_found: int = 0 successful_trades: int = 0 total_profit: Decimal = Decimal('0') start_time: float = field(default_factory=time.time) sandwich_profits: Decimal = Decimal('0') impact_profits: Decimal = Decimal('0') failed_attempts: Dict[str, int] = field(default_factory=dict) strategy_performance: Dict[str, Dict] = field(default_factory=lambda: { 'sandwich': {'success': 0, 'fail': 0, 'avg_profit': Decimal('0'), 'avg_time': 0.0}, 'impact': {'success': 0, 'fail': 0, 'avg_profit': Decimal('0'), 'avg_time': 0.0} }) execution_stats: Dict[str, List[float]] = field(default_factory=lambda: { 'latency': [], 'gas_used': [], 'slippage': [] }) class AdvancedTrading: def __init__(self, trader, config): self.trader = trader self.config = config self.active = False self.metrics = TradingMetrics() self._metrics = { 'trades_executed': 0, 'total_volume': 0.0, 'successful_trades': 0, 'failed_trades': 0, 'average_latency': 0.0, 'peak_latency': 0.0, 'profit_loss': 0.0 } # Initialize process queue self.process_queue = asyncio.Queue(maxsize=int(os.getenv('QUEUE_SIZE', 1000))) # GPU setup with tensor cores enabled self.device = torch.device('cuda') torch.backends.cuda.matmul.allow_tf32 = True torch.backends.cudnn.benchmark = True # Initialize CUDA streams with priorities self.streams = { 'sandwich': torch.cuda.Stream(priority=-2), # Highest priority 'impact': torch.cuda.Stream(priority=-1), # High priority 'compute': torch.cuda.Stream(priority=0) # Normal priority } # Initialize thread pool for CPU tasks cpu_count = psutil.cpu_count(logical=False) self.thread_pool = ThreadPoolExecutor(max_workers=cpu_count) # Setup processing buffers self.setup_processing_buffers() # Initialize opportunity queue with prioritization self.opportunity_queue = asyncio.PriorityQueue() # Load trading parameters from environment variables self._load_trading_parameters() logger.info("Advanced trading initialized with GPU acceleration") def _load_trading_parameters(self): """Load and validate trading parameters""" try: # Load from environment variables self.MIN_PROFIT_PCT = float(os.getenv("MIN_PROFIT_PCT")) self.MAX_PROFIT_PCT = float(os.getenv("MAX_PROFIT_PCT")) self.MAX_PRICE_IMPACT = float(os.getenv("MAX_PRICE_IMPACT")) self.MAX_POSITION_SIZE = float(os.getenv("POSITION_MAX_SIZE")) self.INITIAL_SIZE = float(os.getenv("POSITION_INITIAL_SIZE")) self.GAS_RESERVE = float(os.getenv("GAS_RESERVE")) self.SLIPPAGE = float(os.getenv("MAX_SPREAD")) # Sandwich specific parameters self.MIN_SANDWICH_SIZE = float(os.getenv("SANDWICH_MIN_VICTIM_SIZE")) # Minimum victim tx size in SOL self.MAX_SANDWICH_IMPACT = float(os.getenv("SANDWICH_MAX_IMPACT")) # Maximum impact for sandwich trades self.SANDWICH_PROFIT_THRESHOLD = float(os.getenv("SANDWICH_MIN_PROFIT")) # Minimum profit threshold logger.info("Trading parameters loaded successfully") except KeyError as e: logger.error(f"Missing configuration parameter: {e}") raise def setup_processing_buffers(self): """Initialize optimized GPU processing buffers""" try: gpu_mem = torch.cuda.get_device_properties(0).total_memory buffer_size = min(2048, gpu_mem // (1024 * 1024 * 10)) # Create CPU tensors first self.opportunity_buffer = torch.zeros( (buffer_size, 128), dtype=torch.float16, device='cpu' # Create on CPU first ).pin_memory() # Pin the memory self.impact_buffer = torch.zeros( (buffer_size // 2, 2), dtype=torch.float32, device='cpu' ).pin_memory() self.pattern_buffer = torch.zeros( (1024, 64), dtype=torch.float16, device='cpu' ).pin_memory() # Move to GPU if available if torch.cuda.is_available(): with torch.cuda.stream(self.streams['compute']): self.opportunity_buffer = self.opportunity_buffer.to( device=self.device, non_blocking=True ) self.impact_buffer = self.impact_buffer.to( device=self.device, non_blocking=True ) self.pattern_buffer = self.pattern_buffer.to( device=self.device, non_blocking=True ) torch.cuda.synchronize() logger.debug("GPU buffers initialized") except Exception as e: logger.error(f"Failed to initialize GPU buffers: {e}") raise def _save_metrics(self): """Save trading metrics""" try: metrics_data = { **self._metrics, 'timestamp': time.time(), 'session_duration': time.time() - self.metrics.start_time } logger.info(f"Trading Metrics: {json.dumps(metrics_data, indent=2)}") except Exception as e: logger.error(f"Error saving metrics: {e}") async def cleanup(self): """Cleanup resources properly""" logger.info("Starting cleanup...") try: self.active = False # Clean GPU resources torch.cuda.empty_cache() for stream in self.streams.values(): stream.synchronize() # Save performance metrics self._save_metrics() # Clear process queue while not self.process_queue.empty(): try: self.process_queue.get_nowait() except asyncio.QueueEmpty: break # Close thread pool self.thread_pool.shutdown(wait=True) logger.info("Cleanup completed") except Exception as e: logger.error(f"Cleanup error: {e}") raise # Other methods of the AdvancedTrading class would followIn this code segment, the new code for saving metrics and cleaning up has been integrated into the existing
AdvancedTradingclass. The_save_metricsmethod gathers the metrics and logs them, while thecleanupmethod handles resource cleanup, ensuring that metrics are saved at the end of the session.Feel free to add any additional methods or modify existing ones within the
AdvancedTradingclass as needed! - User
full script please though
- Assistant
Certainly! Here’s the complete
advanced_trading.pyscript with the new Sections added and integrated:import asyncio import base64 import json import logging import os import time from typing import Dict, List, Optional from decimal import Decimal import torch import torch.cuda.amp as amp from dataclasses import dataclass, field import psutil import websockets from concurrent.futures import ThreadPoolExecutor from dotenv import load_dotenv from config_loader import config # Load environment variables from .env file load_dotenv() logging.basicConfig(level=logging.DEBUG, format='%(asctime)s - %(levelname)s - %(message)s') logger = logging.getLogger(__name__) @dataclass class TradingMetrics: messages_received: int = 0 opportunities_found: int = 0 successful_trades: int = 0 total_profit: Decimal = Decimal('0') start_time: float = field(default_factory=time.time) sandwich_profits: Decimal = Decimal('0') impact_profits: Decimal = Decimal('0') failed_attempts: Dict[str, int] = field(default_factory=dict) strategy_performance: Dict[str, Dict] = field(default_factory=lambda: { 'sandwich': {'success': 0, 'fail': 0, 'avg_profit': Decimal('0'), 'avg_time': 0.0}, 'impact': {'success': 0, 'fail': 0, 'avg_profit': Decimal('0'), 'avg_time': 0.0} }) execution_stats: Dict[str, List[float]] = field(default_factory=lambda: { 'latency': [], 'gas_used': [], 'slippage': [] }) class AdvancedTrading: def __init__(self, trader, config): self.trader = trader self.config = config self.active = False self.metrics = TradingMetrics() self._metrics = { 'trades_executed': 0, 'total_volume': 0.0, 'successful_trades': 0, 'failed_trades': 0, 'average_latency': 0.0, 'peak_latency': 0.0, 'profit_loss': 0.0 } # Initialize process queue self.process_queue = asyncio.Queue(maxsize=int(os.getenv('QUEUE_SIZE', 1000))) # GPU setup with tensor cores enabled self.device = torch.device('cuda') torch.backends.cuda.matmul.allow_tf32 = True torch.backends.cudnn.benchmark = True # Initialize CUDA streams with priorities self.streams = { 'sandwich': torch.cuda.Stream(priority=-2), # Highest priority 'impact': torch.cuda.Stream(priority=-1), # High priority 'compute': torch.cuda.Stream(priority=0) # Normal priority } # Initialize thread pool for CPU tasks cpu_count = psutil.cpu_count(logical=False) self.thread_pool = ThreadPoolExecutor(max_workers=cpu_count) # Setup processing buffers self.setup_processing_buffers() # Initialize opportunity queue with prioritization self.opportunity_queue = asyncio.PriorityQueue() # Load trading parameters from environment variables self._load_trading_parameters() logger.info("Advanced trading initialized with GPU acceleration") def _load_trading_parameters(self): """Load and validate trading parameters""" try: # Load from environment variables self.MIN_PROFIT_PCT = float(os.getenv("MIN_PROFIT_PCT")) self.MAX_PROFIT_PCT = float(os.getenv("MAX_PROFIT_PCT")) self.MAX_PRICE_IMPACT = float(os.getenv("MAX_PRICE_IMPACT")) self.MAX_POSITION_SIZE = float(os.getenv("POSITION_MAX_SIZE")) self.INITIAL_SIZE = float(os.getenv("POSITION_INITIAL_SIZE")) self.GAS_RESERVE = float(os.getenv("GAS_RESERVE")) self.SLIPPAGE = float(os.getenv("MAX_SPREAD")) # Sandwich specific parameters self.MIN_SANDWICH_SIZE = float(os.getenv("SANDWICH_MIN_VICTIM_SIZE")) # Minimum victim tx size in SOL self.MAX_SANDWICH_IMPACT = float(os.getenv("SANDWICH_MAX_IMPACT")) # Maximum impact for sandwich trades self.SANDWICH_PROFIT_THRESHOLD = float(os.getenv("SANDWICH_MIN_PROFIT")) # Minimum profit threshold logger.info("Trading parameters loaded successfully") except KeyError as e: logger.error(f"Missing configuration parameter: {e}") raise def setup_processing_buffers(self): """Initialize optimized GPU processing buffers""" try: gpu_mem = torch.cuda.get_device_properties(0).total_memory buffer_size = min(2048, gpu_mem // (1024 * 1024 * 10)) # Create CPU tensors first self.opportunity_buffer = torch.zeros( (buffer_size, 128), dtype=torch.float16, device='cpu' # Create on CPU first ).pin_memory() # Pin the memory self.impact_buffer = torch.zeros( (buffer_size // 2, 2), dtype=torch.float32, device='cpu' ).pin_memory() self.pattern_buffer = torch.zeros( (1024, 64), dtype=torch.float16, device='cpu' ).pin_memory() # Move to GPU if available if torch.cuda.is_available(): with torch.cuda.stream(self.streams['compute']): self.opportunity_buffer = self.opportunity_buffer.to( device=self.device, non_blocking=True ) self.impact_buffer = self.impact_buffer.to( device=self.device, non_blocking=True ) self.pattern_buffer = self.pattern_buffer.to( device=self.device, non_blocking=True ) torch.cuda.synchronize() logger.debug("GPU buffers initialized") except Exception as e: logger.error(f"Failed to initialize GPU buffers: {e}") raise def _save_metrics(self): """Save trading metrics""" try: metrics_data = { **self._metrics, 'timestamp': time.time(), 'session_duration': time.time() - self.metrics.start_time } logger.info(f"Trading Metrics: {json.dumps(metrics_data, indent=2)}") except Exception as e: logger.error(f"Error saving metrics: {e}") async def cleanup(self): """Cleanup resources properly""" logger.info("Starting cleanup...") try: self.active = False # Clean GPU resources torch.cuda.empty_cache() for stream in self.streams.values(): stream.synchronize() # Save performance metrics self._save_metrics() # Clear process queue while not self.process_queue.empty(): try: self.process_queue.get_nowait() except asyncio.QueueEmpty: break # Close thread pool self.thread_pool.shutdown(wait=True) logger.info("Cleanup completed") except Exception as e: logger.error(f"Cleanup error: {e}") raise async def _check_sandwich_opportunity(self, raw_data: bytes) -> Optional[Dict]: """Check for sandwich opportunities with enhanced MEV protection""" try: with torch.cuda.stream(self.streams['sandwich']): # Decode and validate victim transaction victim_tx = await self._decode_victim_transaction(raw_data) if not victim_tx or victim_tx['amount'] < self.MIN_SANDWICH_SIZE: return None # Calculate optimal positions positions = await self._calculate_optimal_positions( victim_tx['pool'], victim_tx['amount'], victim_tx['direction'] ) if not positions: return None # MEV protection check is_safe = await self.trader.mev_protection.verify_sandwich( positions['front_run'], victim_tx['amount'], positions['back_run'] ) if not is_safe: return None # Calculate expected profit profit = await self._calculate_sandwich_profit(positions, victim_tx['pool']) if profit < self.SANDWICH_PROFIT_THRESHOLD: return None return { 'type': 'sandwich', 'profit': profit, 'positions': positions, 'victim': victim_tx, 'pool': victim_tx['pool'], 'confidence': 0.9, 'priority': 1 } except Exception as e: logger.error(f"Sandwich check error: {e}") return None async def _calculate_optimal_positions(self, pool_address: str, victim_amount: float, direction: bool) -> Optional[Dict]: """Calculate optimal front-run and back-run positions""" try: with torch.cuda.stream(self.streams['sandwich']): # Get pool state pool_state = await self.trader.get_pool_state(pool_address) if not pool_state: return None # Convert to tensors for GPU calculation base_reserve = torch.tensor(float(pool_state.base_reserve), device=self.device) quote_reserve = torch.tensor(float(pool_state.quote_reserve), device=self.device) victim_amount_tensor = torch.tensor(victim_amount, device=self.device) # Calculate optimal front-run size with amp.autocast(device_type='cuda'): # Start with 50% of victim tx size front_run_size = victim_amount_tensor * 0.5 # Iteratively optimize front-run size for _ in range(5): # Max 5 iterations # Calculate price impact if direction: # Victim buying new_quote_reserve = quote_reserve + front_run_size new_base_reserve = base_reserve * quote_reserve / new_quote_reserve price_impact = (new_base_reserve / base_reserve - 1).abs() else: # Victim selling new_base_reserve = base_reserve + front_run_size new_quote_reserve = quote_reserve * base_reserve / new_base_reserve price_impact = (new_quote_reserve / quote_reserve - 1).abs() # Adjust size based on impact if price_impact > self.MAX_PRICE_IMPACT: front_run_size *= 0.9 else: break # Calculate back-run size back_run_size = front_run_size * 1.1 # Slightly larger to account for slippage # Calculate minimum output amounts with slippage buffer front_run_min_out = self._calculate_min_output( float(front_run_size), float(base_reserve), float(quote_reserve), self.SLIPPAGE ) back_run_min_out = self._calculate_min_output( float(back_run_size), float(new_base_reserve), float(new_quote_reserve), self.SLIPPAGE ) return { 'front_run': float(front_run_size), 'back_run': float(back_run_size), 'front_run_min_out': front_run_min_out, 'back_run_min_out': back_run_min_out, 'price_impact': float(price_impact) } except Exception as e: logger.error(f"Position calculation error: {e}") return None async def _execute_sandwich_trade(self, opportunity: Dict, priority_fee: int) -> bool: """Execute sandwich trade with precise timing""" try: # Execute front-run with higher priority front_run_fee = int(priority_fee * 1.2) # 20% higher for front-run front_run_result = await self.trader.trading_core.execute_trade( pool_address=opportunity['pool'], amount_in=opportunity['positions']['front_run'], min_amount_out=opportunity['positions']['front_run_min_out'], priority_fee=front_run_fee, compute_limit=600000 # Higher compute limit for complex trades ) if not front_run_result: logger.error("Front-run execution failed") return False # Minimal waiting period for victim transaction await asyncio.sleep(0.05) # 50ms delay # Back-run with lower priority to save on fees back_run_fee = int(priority_fee * 0.8) # 20% lower for back-run back_run_result = await self.trader.trading_core.execute_trade( pool_address=opportunity['pool'], amount_in=opportunity['positions']['back_run'], min_amount_out=opportunity['positions']['back_run_min_out'], priority_fee=back_run_fee ) if not back_run_result: logger.error("Back-run execution failed") return False return True except Exception as e: logger.error(f"Sandwich execution error: {e}") return False def _calculate_min_output(self, amount_in: float, base_reserve: float, quote_reserve: float, slippage: float) -> float: """Calculate minimum output amount with slippage protection""" try: constant_product = base_reserve * quote_reserve new_quote_reserve = quote_reserve + amount_in ideal_output = base_reserve - (constant_product / new_quote_reserve) min_output = ideal_output * (1 - slippage) return max(min_output, 0) except Exception as e: logger.error(f"Min output calculation error: {e}") return 0 async def _optimize_priority_fee(self, opportunity_type: str, profit: float) -> int: """Dynamic priority fee optimization""" try: base_fee = await self.trader.mev_protection.get_priority_fee() multipliers = { 'sandwich': 1.5, # Higher priority for sandwich 'impact': 1.2 # Medium priority for impact trades } fee_multiplier = multipliers.get(opportunity_type, 1.0) if profit > float(os.getenv("PROFIT_THRESHOLD")): fee_multiplier *= float(os.getenv("FEE_MULTIPLIER")) priority_fee = int(base_fee * fee_multiplier) return min( max(priority_fee, int(os.getenv("MIN_PRIORITY_FEE"))), int(os.getenv("MAX_PRIORITY_FEE")) ) except Exception as e: logger.error(f"Fee optimization error: {e}") return int(os.getenv("DEFAULT_PRIORITY_FEE")) async def execute_opportunity(self, opportunity: Dict) -> bool: """Execute opportunity with dynamic fee optimization""" start_time = time.time() try: if not await self._validate_execution_conditions(opportunity): return False priority_fee = await self._optimize_priority_fee( opportunity['type'], opportunity['profit'] ) if opportunity['type'] == 'sandwich': result = await self._execute_sandwich_trade(opportunity, priority_fee) else: # impact trade result = await self._execute_impact_trade(opportunity, priority_fee) if result: execution_time = time.time() - start_time await self._update_execution_stats(opportunity, execution_time) self.metrics.successful_trades += 1 if opportunity['type'] == 'sandwich': self.metrics.sandwich_profits += Decimal(str(opportunity['profit'])) else: self.metrics.impact_profits += Decimal(str(opportunity['profit'])) logger.info(f"Trade successful: {opportunity['type']}, Profit: {opportunity['profit']:.4f}") return True return False except Exception as e: logger.error(f"Execution error: {e}") execution_time = time.time() - start_time await self._update_failed_execution_stats(opportunity, execution_time) return False async def start_websocket_monitoring(self): """High-performance WebSocket monitoring""" self.active = True logger.info("Starting advanced trading monitor...") while self.active: try: async with websockets.connect( os.getenv("HELIUS_WS_URL"), ping_interval=int(os.getenv("WS_PING_INTERVAL")), ping_timeout=int(os.getenv("WS_PING_TIMEOUT")), close_timeout=int(os.getenv("WS_CLOSE_TIMEOUT")), max_size=int(os.getenv("WS_MAX_MESSAGE_SIZE")) ) as ws: await ws.send(json.dumps({ "jsonrpc": "2.0", "id": 1, "method": "logsSubscribe", "params": [ {"mentions": [self.config['programs']['raydium']]}, {"commitment": "processed"} ] })) processors = [ self._process_messages(ws), self._monitor_network_conditions(), self._optimize_execution_parameters() ] await asyncio.gather(*processors) except websockets.exceptions.ConnectionClosed: logger.warning("WebSocket connection closed. Reconnecting...") await asyncio.sleep(0.1) except Exception as e: logger.error(f"WebSocket error: {e}") await asyncio.sleep(0.1) async def _process_messages(self, ws): """Process messages with GPU acceleration""" while self.active: try: msg = await ws.recv() self.metrics.messages_received += 1 with amp.autocast(device_type='cuda'): opportunities = await self._parallel_opportunity_check(msg) if opportunities: await self._execute_best_opportunity(opportunities) except Exception as e: logger.error(f"Message processing error: {e}") @torch.amp.autocast('cuda') async def _parallel_opportunity_check(self, msg: str) -> List[Dict]: """Check for sandwich and price impact opportunities in parallel""" try: data = json.loads(msg) if 'params' not in data: return [] logs = data['params'].get('result', {}).get('value', {}).get('logs', []) opportunities = [] for log in logs: if "ray_log:" not in log: continue raw_data = base64.b64decode(log.split('ray_log: ')[1].strip()) # Parallel opportunity checking with torch.cuda.stream(self.streams['compute']): results = await asyncio.gather( self._check_sandwich_opportunity(raw_data), self._check_price_impact_opportunity(raw_data) ) valid_results = [r for r in results if r and self._validate_opportunity(r)] if valid_results: opportunities.extend(self._calculate_opportunity_scores(valid_results)) return opportunities except Exception as e: logger.error(f"Parallel check error: {e}") return [] async def _check_price_impact_opportunity(self, raw_data: bytes) -> Optional[Dict]: """Check for price impact opportunities""" try: with torch.cuda.stream(self.streams['impact']): # Extract transaction data tx_data = await self._process_trade_data(raw_data) if not tx_data: return None # Calculate impact and profit impact = await self._calculate_price_impact( tx_data['amount_in'], tx_data['amount_out'], tx_data['pool'] ) if impact > self.MAX_PRICE_IMPACT: return None profit = await self._calculate_impact_profit(tx_data, impact) if profit < self.MIN_PROFIT_PCT: return None return { 'type': 'impact', 'profit': profit, 'amount': tx_data['amount_in'], 'pool': tx_data['pool'], 'impact': impact, 'confidence': 0.85, 'priority': 2 } except Exception as e: logger.error(f"Impact check error: {e}") return NoneKey Features of the Script:
-
Trading Metrics: The
TradingMetricsdata class encapsulates various trading metrics needed to monitor and improve trading strategy performance. -
AdvancedTrading Class: Handles all key trading operations including loading parameters, processing opportunities, executing trades, and cleanup.
-
Asynchronous Monitoring: Uses WebSocket connections for monitoring trading opportunities, allowing for responsive and real-time trading adjustments.
-
GPU Optimization: Utilizes GPU acceleration for heavy computations, particularly with PyTorch for tensor calculations.
-
Metrics Logging: The
_save_metricsmethod collects metrics throughout trades and logs them for future reference. -
Cleanup Methods: Cleans up resources effectively, ensuring that GPU memory is managed correctly and that logging happens at the end of a session.
Modify the methods according to your own trading logic and integrate them with your different components as necessary. If you have more specific functionalities you want to add or any assumptions I should consider, feel free to let me know!
-
- User
def init(self): # Your existing initialization...
# Add all required queues self.mempool_queue = asyncio.Queue(maxsize=int(os.getenv('QUEUE_SIZE', 1000))) self.price_queue = asyncio.Queue(maxsize=int(os.getenv('QUEUE_SIZE', 1000))) self.process_queue = asyncio.Queue(maxsize=int(os.getenv('QUEUE_SIZE', 1000))) # Add price tracking self.price_data = None self.last_price_update = 0 # Initialize metrics self._metrics = { 'messages_processed': 0, 'trades_detected': 0, 'errors': 0, 'latency': 0.0 }async def _clear_queues(self): """Clear all message queues""" try: logger.debug("Clearing all message queues...") queues = [ self.mempool_queue, self.price_queue, self.process_queue ]
for queue in queues: while not queue.empty(): try: queue.get_nowait() except asyncio.QueueEmpty: break logger.info("All queues cleared") except Exception as e: logger.error(f"Error clearing queues: {e}") <---- gets added to >>>>> import asyncioimport logging import json import websockets from typing import Dict, Optional from dataclasses import dataclass, field import time import torch import torch.cuda.amp as amp import numpy as np import base64 from concurrent.futures import ThreadPoolExecutor import os from dotenv import load_dotenv from config_loader import config from solana_rust_bot import TradeParams, TradingCore
Load environment variables
load_dotenv()
Configure logging
logging.basicConfig( level=logging.DEBUG, format='%(asctime)s.%(msecs)03d [%(levelname)s] [%(filename)s:%(lineno)d] %(message)s', datefmt='%Y-%m-%d %H:%M:%S' ) logger = logging.getLogger(name)
@dataclass class PriceData: price: float volume: float bid: float ask: float timestamp: float spread: float = field(init=False)
def __post_init__(self): self.spread = (self.ask - self.bid) / self.price if self.price > 0 else float('inf')@dataclass class ConnectionMetrics: messages_received: int = 0 reconnect_count: int = 0 total_latency: float = 0.0 peak_latency: float = 0.0 errors: int = 0 mempool_hits: int = 0 processed_txs: int = 0 start_time: float = field(default_factory=time.time)
class HeliusWebSocketClient: def init(self): """Initialize WebSocket client with enhanced configuration""" logger.info("Initializing enhanced WebSocket client...")
# Core WebSocket Configuration self.helius_ws = os.getenv('WS_URL') self.helius_key = os.getenv('HELIUS_API_KEY') self.quicknode_ws = os.getenv('QUICKNODE_WEBSOCKET') self.quicknode_key = os.getenv('QUICKNODE_API_KEY') # WebSocket Parameters self.ping_interval = float(os.getenv('WS_PING_INTERVAL', 30)) self.ping_timeout = float(os.getenv('WS_PING_TIMEOUT', 10)) self.close_timeout = float(os.getenv('WS_CLOSE_TIMEOUT', 60)) self.max_message_size = int(os.getenv('WS_MAX_MESSAGE_SIZE', 16777216)) self.reconnect_delay = float(os.getenv('WS_RECONNECT_DELAY', 0.1)) # Program IDs self.RAYDIUM_PROGRAM = os.getenv('RAYDIUM_PROGRAM_ID') self.SOL_WSOL_MARKET = os.getenv('SOL_WSOL_AMM_ID') self.AMM_ID = os.getenv('SOL_WSOL_AMM_ID') # Initialize GPU settings if torch.cuda.is_available(): torch.backends.cuda.matmul.allow_tf32 = os.getenv('ALLOW_TF32', 'true').lower() == 'true' torch.backends.cudnn.benchmark = os.getenv('CUDNN_BENCHMARK', 'true').lower() == 'true' # Configure CUDA streams self.streams = { 'process': torch.cuda.Stream(priority=int(os.getenv('GPU_PROCESS_STREAMS', -2))), 'compute': torch.cuda.Stream(priority=int(os.getenv('GPU_COMPUTE_STREAMS', -1))), 'update': torch.cuda.Stream(priority=int(os.getenv('GPU_UPDATE_STREAMS', 0))) } # Thread pool for background tasks self.thread_pool = ThreadPoolExecutor(max_workers=2) # Add queues self.mempool_queue = asyncio.Queue(maxsize=int(os.getenv('QUEUE_SIZE', 1000))) self.price_queue = asyncio.Queue(maxsize=int(os.getenv('QUEUE_SIZE', 1000))) self.message_queue = asyncio.Queue(maxsize=int(os.getenv('QUEUE_SIZE', 1000))) # Price tracking self.price_data = None self.last_price_update = 0 # Add MEV protection stats self._stats = { 'transactions_analyzed': 0, 'opportunities_detected': 0, 'successful_executions': 0, 'failed_executions': 0, 'total_profit': 0.0 } logger.info("WebSocket client initialized") def is_connected(self) -> bool: """Check if websocket is connected""" return (hasattr(self, 'ws_helius') and self.ws_helius is not None and not self.ws_helius.closed) async def start(self): """Start WebSocket connections and processors""" logger.debug("[INFO] Starting Helius WebSocket Client...") try: self.active = True # Start connections and processors tasks = [ asyncio.create_task(self._maintain_helius_connection()), asyncio.create_task(self._maintain_quicknode_connection()), asyncio.create_task(self._process_messages()), asyncio.create_task(self._monitor_connections()) ] # Wait for connections to establish await asyncio.gather(*tasks) except Exception as e: logger.error(f"Failed to start WebSocket client: {e}") self.active = False raise async def _maintain_helius_connection(self): """Maintain Helius WebSocket connection""" reconnect_delay = 0.1 max_reconnect_delay = 5.0 while self.active: try: async with websockets.connect( self.helius_ws, ping_interval=self.ping_interval, ping_timeout=self.ping_timeout, close_timeout=self.close_timeout, max_size=self.max_message_size ) as ws: self.ws_helius = ws logger.info("Connected to Helius WebSocket") reconnect_delay = 0.1 # Reset delay on successful connection # Subscribe to required programs and accounts await self._subscribe_helius() # Process messages async for message in ws: if not self.active: break try: data = json.loads(message) await self._handle_helius_message(data) except Exception as e: logger.error(f"Error processing Helius message: {e}") except websockets.exceptions.ConnectionClosed: logger.warning("Helius connection closed") self.ws_helius = None except Exception as e: logger.error(f"Helius connection error: {e}") self.metrics.errors += 1 if self.active: await asyncio.sleep(reconnect_delay) reconnect_delay = min(reconnect_delay * 2, max_reconnect_delay) self.metrics.reconnect_count += 1 async def _maintain_quicknode_connection(self): """Maintain QuickNode WebSocket connection""" reconnect_delay = 0.1 max_reconnect_delay = 5.0 while self.active: try: async with websockets.connect( self.quicknode_ws, ping_interval=self.ping_interval, ping_timeout=self.ping_timeout, close_timeout=self.close_timeout, max_size=self.max_message_size ) as ws: self.ws_quicknode = ws logger.info("Connected to QuickNode WebSocket") reconnect_delay = 0.1 # Reset delay on successful connection # Subscribe to market data await self._subscribe_quicknode() # Process messages async for message in ws: if not self.active: break try: data = json.loads(message) await self._handle_quicknode_message(data) except Exception as e: logger.error(f"Error processing QuickNode message: {e}") except websockets.exceptions.ConnectionClosed: logger.warning("QuickNode connection closed") self.ws_quicknode = None except Exception as e: logger.error(f"QuickNode connection error: {e}") self.metrics.errors += 1 if self.active: await asyncio.sleep(reconnect_delay) reconnect_delay = min(reconnect_delay * 2, max_reconnect_delay) self.metrics.reconnect_count += 1 async def _subscribe_helius(self): """Subscribe to Helius data streams""" logger.debug("[INFO] Subscribing to Helius streams...") try: # Subscribe to Raydium program await self.ws_helius.send(json.dumps({ "jsonrpc": "2.0", "id": "raydium_sub", "method": "programSubscribe", "params": [ self.RAYDIUM_PROGRAM, {"encoding": "base64", "commitment": "processed"} ] })) logger.info(f"[INFO] Subscribed to Raydium program: {self.RAYDIUM_PROGRAM}") # Subscribe to SOL/wSOL market await self.ws_helius.send(json.dumps({ "jsonrpc": "2.0", "id": "market_sub", "method": "accountSubscribe", "params": [ self.SOL_WSOL_MARKET, {"encoding": "base64", "commitment": "processed"} ] })) logger.info(f"[INFO] Subscribed to market: {self.SOL_WSOL_MARKET}") # Subscribe to AMM account await self.ws_helius.send(json.dumps({ "jsonrpc": "2.0", "id": "amm_sub", "method": "accountSubscribe", "params": [ self.AMM_ID, {"encoding": "base64", "commitment": "processed"} ] })) logger.info(f"[INFO] Subscribed to AMM account: {self.AMM_ID}") logger.info("Helius subscriptions completed") except Exception as e: logger.error(f"Helius subscription error: {e}") raise async def _subscribe_quicknode(self): """Subscribe to QuickNode market data""" logger.debug("[INFO] Subscribing to QuickNode market data...") try: await self.ws_quicknode.send(json.dumps({ "jsonrpc": "2.0", "id": "market_data", "method": "subscribeToMarket", "params": [ self.SOL_WSOL_MARKET, { "commitment": "processed", "encoding": "jsonParsed", "levels": 20 # Order book depth } ] })) logger.info(f"[INFO] Subscribed to QuickNode market: {self.SOL_WSOL_MARKET}") except Exception as e: logger.error(f"QuickNode subscription error: {e}") raise async def _handle_helius_message(self, message: Dict): try: if 'method' in message and message['method'] == 'programNotification': program_data = message['params']['result'] if self.RAYDIUM_PROGRAM in str(program_data): decoded_data = base64.b64decode(program_data['value']['account']['data'][0]) if decoded_data[0] == 9: # Swap instruction tx_data = { 'pool_id': program_data['value']['pubkey'], 'amount_in': int.from_bytes(decoded_data[1:9], 'little') / 1e9, 'timestamp': time.time_ns() } # Get current pool state pool_state = self.impact_calculator.pool_states.get(tx_data['pool_id']) if pool_state: # Analyze for MEV opportunity impact_result = await self.impact_calculator.analyze_mempool_tx( tx_data, pool_state ) if (impact_result.is_valid and impact_result.confidence > self.impact_calculator.config.CONFIDENCE_THRESHOLD): # Execute through Rust backend params = TradeParams( pool_address=tx_data['pool_id'], amount_in=impact_result.optimal_size, min_amount_out=impact_result.min_amount_out, wsol_account=os.getenv('WSOL_TOKEN_ACCOUNT') ) signature = self.trading_core.execute_trade(params) if signature: logger.info(f"MEV trade executed: {signature}") except Exception as e: logger.error(f"Helius message handling error: {e}") async def _handle_quicknode_message(self, message: Dict): """Process QuickNode WebSocket messages""" try: logger.debug(f"[DEBUG] Handling QuickNode message: {message}") if 'method' in message and message['method'] == 'marketData': market_data = message['params']['data'] # Update price data if valid price_data = PriceData( price=float(market_data['lastPrice']), volume=float(market_data['volume24h']), bid=float(market_data['bestBid']), ask=float(market_data['bestAsk']), timestamp=time.time() ) self.price_data = price_data self.last_price_update = time.time() await self.price_queue.put(price_data) logger.debug(f"Price update: {self.price_data}") self.metrics.messages_received += 1 except Exception as e: logger.error(f"Error handling QuickNode message: {e}") self.metrics.errors += 1 async def _handle_account_update(self, account_data: Dict): """Process account update messages""" try: logger.debug(f"[DEBUG] Handling account update for: {account_data}") account = account_data.get('pubkey') if not account: return if account == self.SOL_WSOL_MARKET: await self.process_queue.put({ 'type': 'market_update', 'data': account_data }) elif account == self.AMM_ID: await self.process_queue.put({ 'type': 'amm_update', 'data': account_data }) except Exception as e: logger.error(f"Error handling account update: {e}") async def _process_messages(self): """Process messages from all queues""" while self.active: try: # Process with timeout to prevent blocking await asyncio.wait_for(self._process_message_batch(), timeout=0.1) except asyncio.TimeoutError: continue except Exception as e: logger.error(f"Message processing error: {e}") await asyncio.sleep(0.1) async def _process_message_batch(self): """Process a batch of messages from queues""" try: # Process up to 100 messages per batch for _ in range(100): if not self.mempool_queue.empty(): message = await self.mempool_queue.get() self.metrics.processed_txs += 1 await self.process_queue.put({ 'type': 'mempool', 'data': message }) logger.debug("Processed mempool message") if not self.price_queue.empty(): price_data = await self.price_queue.get() await self.process_queue.put({ 'type': 'price', 'data': price_data }) logger.debug("Processed price update") await asyncio.sleep(0) # Yield to other tasks except Exception as e: logger.error(f"Batch processing error: {e}") async def _monitor_connections(self): """Monitor WebSocket connections health""" logger.debug("[INFO] Starting connection monitoring...") while self.active: try: current_time = time.time() # Check Helius connection if self.ws_helius and self.ws_helius.closed: logger.warning("Helius connection closed") self.ws_helius = None # Check QuickNode connection if self.ws_quicknode and self.ws_quicknode.closed: logger.warning("QuickNode connection closed") self.ws_quicknode = None # Check price data freshness if current_time - self.last_price_update > 5: logger.warning("Price data stale") await asyncio.sleep(1) except Exception as e: logger.error(f"Connection monitoring error: {e}") await asyncio.sleep(1) async def cleanup(self): """Cleanup WebSocket connections and resources""" logger.info("Starting WebSocket cleanup...") self.active = False try: # Close WebSocket connections if self.ws_helius: await self.ws_helius.close() logger.info("Helius WebSocket closed") if self.ws_quicknode: await self.ws_quicknode.close() logger.info("QuickNode WebSocket closed") # Clear queues await self._clear_queues() # Clean up GPU resources if torch.cuda.is_available(): try: torch.cuda.synchronize() for stream in self.streams.values(): stream.synchronize() torch.cuda.empty_cache() logger.info("GPU resources cleaned up") except Exception as e: logger.error(f"GPU cleanup error: {e}") # Shutdown thread pool self.thread_pool.shutdown(wait=False) # Log final metrics final_metrics = await self.get_connection_status() logger.info(f"Final WebSocket metrics: {json.dumps(final_metrics, indent=2)}") except Exception as e: logger.error(f"Error during cleanup: {e}") raise finally: logger.info("WebSocket cleanup completed") async def _clear_queues(self): """Clear all message queues""" try: logger.debug("Clearing all message queues...") queues = [self.mempool_queue, self.price_queue, self.process_queue] for queue in queues: while not queue.empty(): try: queue.get_nowait() except asyncio.QueueEmpty: break logger.info("All queues cleared") except Exception as e: logger.error(f"Error clearing queues: {e}") async def get_current_price(self) -> Optional[PriceData]: """Get current price data with freshness check""" if self.price_data and time.time() - self.last_price_update < 5: logger.debug(f"[DEBUG] Current price data retrieved: {self.price_data}") return self.price_data logger.debug("[DEBUG] No fresh price data available.") return None async def get_connection_status(self) -> Dict: """Get current connection status""" status = { 'helius_connected': bool(self.ws_helius and not self.ws_helius.closed), 'quicknode_connected': bool(self.ws_quicknode and not self.ws_quicknode.closed), 'price_data_fresh': bool(self.price_data and time.time() - self.last_price_update < 5), 'mempool_queue_size': self.mempool_queue.qsize(), 'price_queue_size': self.price_queue.qsize(), 'process_queue_size': self.process_queue.qsize(), 'metrics': { 'messages_received': self.metrics.messages_received, 'reconnect_count': self.metrics.reconnect_count, 'errors': self.metrics.errors, 'mempool_hits': self.metrics.mempool_hits, 'processed_txs': self.metrics.processed_txs, 'uptime': time.time() - self.metrics.start_time } } logger.debug(f"[DEBUG] Connection status: {status}") return status async def get_market_depth(self) -> Dict: """Get current market depth""" logger.debug("[INFO] Getting current market depth...") try: if not self.ws_quicknode: logger.warning("[WARNING] QuickNode WebSocket is not connected.") return {} await self.ws_quicknode.send(json.dumps({ "jsonrpc": "2.0", "id": "depth", "method": "getOrderbook", "params": [ self.SOL_WSOL_MARKET, 20 # Depth levels ] })) response = await self.ws_quicknode.recv() data = json.loads(response) if 'result' in data: logger.debug("[DEBUG] Market depth data retrieved.") return { 'bids': data['result']['bids'], 'asks': data['result']['asks'], 'timestamp': time.time() } logger.warning("[WARNING] No market depth data found in response.") return {} except Exception as e: logger.error(f"Error getting market depth: {e}") return {} def _calculate_latency(self, start_time: float) -> float: """Calculate message processing latency""" latency = time.time() - start_time self.metrics.total_latency += latency self.metrics.peak_latency = max(self.metrics.peak_latency, latency) logger.debug(f"[DEBUG] Latency calculated: {latency:.3f}s") return latency def _save_statistics(self): """Save MEV protection statistics""" try: logger.info(f"MEV Protection Stats: {json.dumps(self._stats, indent=2)}") except Exception as e: logger.error(f"Error saving statistics: {e}") def _save_metrics(self): """Save trading metrics""" try: metrics = { 'price_updates': len(self.price_queue._queue), 'mempool_updates': len(self.mempool_queue._queue), 'last_price': self.price_data.price if self.price_data else None, 'last_update': self.last_price_update } logger.info(f"Trading Metrics: {json.dumps(metrics, indent=2)}") except Exception as e: logger.error(f"Error saving metrics: {e}") def _log_error(self, error: Exception, context: str = ""): """Log error with context""" error_msg = f"{context} error: {str(error)}" if context else str(error) logger.error(error_msg) self.metrics.errors += 1Example usage:
if name == "main": client = HeliusWebSocketClient() try: asyncio.run(client.start()) except Exception as e: logger.error(f"Error running WebSocket Client: {e}") finally: asyncio.run(client.cleanup())
- Assistant
Certainly! Below is the complete
helius_websocket_client.pyscript incorporating the new features you've outlined, such as the initialization of multiple queues and the addition of methods for clearing the queues and managing WebSocket connections.Here's the full code:
import asyncio import logging import json import websockets from typing import Dict, Optional from dataclasses import dataclass, field import time import torch import torch.cuda.amp as amp import base64 from concurrent.futures import ThreadPoolExecutor import os from dotenv import load_dotenv from config_loader import config from solana_rust_bot import TradeParams, TradingCore # Load environment variables load_dotenv() # Configure logging logging.basicConfig( level=logging.DEBUG, format='%(asctime)s.%(msecs)03d [%(levelname)s] [%(filename)s:%(lineno)d] %(message)s', datefmt='%Y-%m-%d %H:%M:%S' ) logger = logging.getLogger(__name__) @dataclass class PriceData: price: float volume: float bid: float ask: float timestamp: float spread: float = field(init=False) def __post_init__(self): self.spread = (self.ask - self.bid) / self.price if self.price > 0 else float('inf') @dataclass class ConnectionMetrics: messages_received: int = 0 reconnect_count: int = 0 total_latency: float = 0.0 peak_latency: float = 0.0 errors: int = 0 mempool_hits: int = 0 processed_txs: int = 0 start_time: float = field(default_factory=time.time) class HeliusWebSocketClient: def __init__(self): """Initialize WebSocket client with enhanced configuration""" logger.info("Initializing enhanced WebSocket client...") # Core WebSocket Configuration self.helius_ws = os.getenv('WS_URL') self.helius_key = os.getenv('HELIUS_API_KEY') self.quicknode_ws = os.getenv('QUICKNODE_WEBSOCKET') self.quicknode_key = os.getenv('QUICKNODE_API_KEY') # WebSocket Parameters self.ping_interval = float(os.getenv('WS_PING_INTERVAL', 30)) self.ping_timeout = float(os.getenv('WS_PING_TIMEOUT', 10)) self.close_timeout = float(os.getenv('WS_CLOSE_TIMEOUT', 60)) self.max_message_size = int(os.getenv('WS_MAX_MESSAGE_SIZE', 16777216)) self.reconnect_delay = float(os.getenv('WS_RECONNECT_DELAY', 0.1)) # Program IDs self.RAYDIUM_PROGRAM = os.getenv('RAYDIUM_PROGRAM_ID') self.SOL_WSOL_MARKET = os.getenv('SOL_WSOL_AMM_ID') self.AMM_ID = os.getenv('SOL_WSOL_AMM_ID') # Initialize GPU settings if torch.cuda.is_available(): torch.backends.cuda.matmul.allow_tf32 = os.getenv('ALLOW_TF32', 'true').lower() == 'true' torch.backends.cudnn.benchmark = os.getenv('CUDNN_BENCHMARK', 'true').lower() == 'true' # Configure CUDA streams self.streams = { 'process': torch.cuda.Stream(priority=int(os.getenv('GPU_PROCESS_STREAMS', -2))), 'compute': torch.cuda.Stream(priority=int(os.getenv('GPU_COMPUTE_STREAMS', -1))), 'update': torch.cuda.Stream(priority=int(os.getenv('GPU_UPDATE_STREAMS', 0))) } # Thread pool for background tasks self.thread_pool = ThreadPoolExecutor(max_workers=2) # Add all required queues self.mempool_queue = asyncio.Queue(maxsize=int(os.getenv('QUEUE_SIZE', 1000))) self.price_queue = asyncio.Queue(maxsize=int(os.getenv('QUEUE_SIZE', 1000))) self.process_queue = asyncio.Queue(maxsize=int(os.getenv('QUEUE_SIZE', 1000))) # Price tracking self.price_data = None self.last_price_update = 0 # Add MEV protection stats self._stats = { 'transactions_analyzed': 0, 'opportunities_detected': 0, 'successful_executions': 0, 'failed_executions': 0, 'total_profit': 0.0 } # Initialize metrics self.metrics = ConnectionMetrics() logger.info("WebSocket client initialized") def is_connected(self) -> bool: """Check if websocket is connected""" return (hasattr(self, 'ws_helius') and self.ws_helius is not None and not self.ws_helius.closed) async def start(self): """Start WebSocket connections and processors""" logger.debug("[INFO] Starting Helius WebSocket Client...") try: self.active = True # Start connections and processors tasks = [ asyncio.create_task(self._maintain_helius_connection()), asyncio.create_task(self._maintain_quicknode_connection()), asyncio.create_task(self._process_messages()), asyncio.create_task(self._monitor_connections()) ] # Wait for connections to establish await asyncio.gather(*tasks) except Exception as e: logger.error(f"Failed to start WebSocket client: {e}") self.active = False raise async def _maintain_helius_connection(self): """Maintain Helius WebSocket connection""" reconnect_delay = 0.1 max_reconnect_delay = 5.0 while self.active: try: async with websockets.connect( self.helius_ws, ping_interval=self.ping_interval, ping_timeout=self.ping_timeout, close_timeout=self.close_timeout, max_size=self.max_message_size ) as ws: self.ws_helius = ws logger.info("Connected to Helius WebSocket") reconnect_delay = 0.1 # Reset delay on successful connection # Subscribe to required programs and accounts await self._subscribe_helius() # Process messages async for message in ws: if not self.active: break try: data = json.loads(message) await self._handle_helius_message(data) except Exception as e: logger.error(f"Error processing Helius message: {e}") except websockets.exceptions.ConnectionClosed: logger.warning("Helius connection closed") self.ws_helius = None except Exception as e: logger.error(f"Helius connection error: {e}") self.metrics.errors += 1 if self.active: await asyncio.sleep(reconnect_delay) reconnect_delay = min(reconnect_delay * 2, max_reconnect_delay) self.metrics.reconnect_count += 1 async def _maintain_quicknode_connection(self): """Maintain QuickNode WebSocket connection""" reconnect_delay = 0.1 max_reconnect_delay = 5.0 while self.active: try: async with websockets.connect( self.quicknode_ws, ping_interval=self.ping_interval, ping_timeout=self.ping_timeout, close_timeout=self.close_timeout, max_size=self.max_message_size ) as ws: self.ws_quicknode = ws logger.info("Connected to QuickNode WebSocket") reconnect_delay = 0.1 # Reset delay on successful connection # Subscribe to market data await self._subscribe_quicknode() # Process messages async for message in ws: if not self.active: break try: data = json.loads(message) await self._handle_quicknode_message(data) except Exception as e: logger.error(f"Error processing QuickNode message: {e}") except websockets.exceptions.ConnectionClosed: logger.warning("QuickNode connection closed") self.ws_quicknode = None except Exception as e: logger.error(f"QuickNode connection error: {e}") self.metrics.errors += 1 if self.active: await asyncio.sleep(reconnect_delay) reconnect_delay = min(reconnect_delay * 2, max_reconnect_delay) self.metrics.reconnect_count += 1 async def _subscribe_helius(self): """Subscribe to Helius data streams""" logger.debug("[INFO] Subscribing to Helius streams...") try: # Subscribe to Raydium program await self.ws_helius.send(json.dumps({ "jsonrpc": "2.0", "id": "raydium_sub", "method": "programSubscribe", "params": [ self.RAYDIUM_PROGRAM, {"encoding": "base64", "commitment": "processed"} ] })) logger.info(f"[INFO] Subscribed to Raydium program: {self.RAYDIUM_PROGRAM}") # Subscribe to SOL/wSOL market await self.ws_helius.send(json.dumps({ "jsonrpc": "2.0", "id": "market_sub", "method": "accountSubscribe", "params": [ self.SOL_WSOL_MARKET, {"encoding": "base64", "commitment": "processed"} ] })) logger.info(f"[INFO] Subscribed to market: {self.SOL_WSOL_MARKET}") # Subscribe to AMM account await self.ws_helius.send(json.dumps({ "jsonrpc": "2.0", "id": "amm_sub", "method": "accountSubscribe", "params": [ self.AMM_ID, {"encoding": "base64", "commitment": "processed"} ] })) logger.info(f"[INFO] Subscribed to AMM account: {self.AMM_ID}") logger.info("Helius subscriptions completed") except Exception as e: logger.error(f"Helius subscription error: {e}") raise async def _subscribe_quicknode(self): """Subscribe to QuickNode market data""" logger.debug("[INFO] Subscribing to QuickNode market data...") try: await self.ws_quicknode.send(json.dumps({ "jsonrpc": "2.0", "id": "market_data", "method": "subscribeToMarket", "params": [ self.SOL_WSOL_MARKET, { "commitment": "processed", "encoding": "jsonParsed", "levels": 20 # Order book depth } ] })) logger.info(f"[INFO] Subscribed to QuickNode market: {self.SOL_WSOL_MARKET}") except Exception as e: logger.error(f"QuickNode subscription error: {e}") raise async def _handle_helius_message(self, message: Dict): try: if 'method' in message and message['method'] == 'programNotification': program_data = message['params']['result'] if self.RAYDIUM_PROGRAM in str(program_data): decoded_data = base64.b64decode(program_data['value']['account']['data'][0]) if decoded_data[0] == 9: # Swap instruction tx_data = { 'pool_id': program_data['value']['pubkey'], 'amount_in': int.from_bytes(decoded_data[1:9], 'little') / 1e9, 'timestamp': time.time_ns() } # Get current pool state pool_state = self.impact_calculator.pool_states.get(tx_data['pool_id']) if pool_state: # Analyze for MEV opportunity impact_result = await self.impact_calculator.analyze_mempool_tx( tx_data, pool_state ) if (impact_result.is_valid and impact_result.confidence > self.impact_calculator.config.CONFIDENCE_THRESHOLD): # Execute through Rust backend params = TradeParams( pool_address=tx_data['pool_id'], amount_in=impact_result.optimal_size, min_amount_out=impact_result.min_amount_out, wsol_account=os.getenv('WSOL_TOKEN_ACCOUNT') ) signature = self.trading_core.execute_trade(params) if signature: logger.info(f"MEV trade executed: {signature}") except Exception as e: logger.error(f"Helius message handling error: {e}") async def _handle_quicknode_message(self, message: Dict): """Process QuickNode WebSocket messages""" try: logger.debug(f"[DEBUG] Handling QuickNode message: {message}") if 'method' in message and message['method'] == 'marketData': market_data = message['params']['data'] # Update price data if valid price_data = PriceData( price=float(market_data['lastPrice']), volume=float(market_data['volume24h']), bid=float(market_data['bestBid']), ask=float(market_data['bestAsk']), timestamp=time.time() ) self.price_data = price_data self.last_price_update = time.time() await self.price_queue.put(price_data) logger.debug(f"Price update: {self.price_data}") self.metrics.messages_received += 1 except Exception as e: logger.error(f"Error handling QuickNode message: {e}") self.metrics.errors += 1 async def _handle_account_update(self, account_data: Dict): """Process account update messages""" try: logger.debug(f"[DEBUG] Handling account update for: {account_data}") account = account_data.get('pubkey') if not account: return if account == self.SOL_WSOL_MARKET: await self.process_queue.put({ 'type': 'market_update', 'data': account_data }) elif account == self.AMM_ID: await self.process_queue.put({ 'type': 'amm_update', 'data': account_data }) except Exception as e: logger.error(f"Error handling account update: {e}") async def _process_messages(self): """Process messages from all queues""" while self.active: try: # Process with timeout to prevent blocking await asyncio.wait_for(self._process_message_batch(), timeout=0.1) except asyncio.TimeoutError: continue except Exception as e: logger.error(f"Message processing error: {e}") await asyncio.sleep(0.1) async def _process_message_batch(self): """Process a batch of messages from queues""" try: # Process up to 100 messages per batch for _ in range(100): if not self.mempool_queue.empty(): message = await self.mempool_queue.get() self.metrics.processed_txs += 1 await self.process_queue.put({ 'type': 'mempool', 'data': message }) logger.debug("Processed mempool message") if not self.price_queue.empty(): price_data = await self.price_queue.get() await self.process_queue.put({ 'type': 'price', 'data': price_data }) logger.debug("Processed price update") await asyncio.sleep(0) # Yield to other tasks except Exception as e: logger.error(f"Batch processing error: {e}") async def _monitor_connections(self): """Monitor WebSocket connections health""" logger.debug("[INFO] Starting connection monitoring...") while self.active: try: current_time = time.time() # Check Helius connection if self.ws_helius and self.ws_helius.closed: logger.warning("Helius connection closed") self.ws_helius = None # Check QuickNode connection if self.ws_quicknode and self.ws_quicknode.closed: logger.warning("QuickNode connection closed") self.ws_quicknode = None # Check price data freshness if current_time - self.last_price_update > 5: logger.warning("Price data stale") await asyncio.sleep(1) except Exception as e: logger.error(f"Connection monitoring error: {e}") await asyncio.sleep(1) async def cleanup(self): """Cleanup WebSocket connections and resources""" logger.info("Starting WebSocket cleanup...") self.active = False try: # Close WebSocket connections if self.ws_helius: await self.ws_helius.close() logger.info("Helius WebSocket closed") if self.ws_quicknode: await self.ws_quicknode.close() logger.info("QuickNode WebSocket closed") # Clear queues await self._clear_queues() # Clean up GPU resources if torch.cuda.is_available(): try: torch.cuda.synchronize() for stream in self.streams.values(): stream.synchronize() torch.cuda.empty_cache() logger.info("GPU resources cleaned up") except Exception as e: logger.error(f"GPU cleanup error: {e}") # Shutdown thread pool self.thread_pool.shutdown(wait=False) # Log final metrics final_metrics = await self.get_connection_status() logger.info(f"Final WebSocket metrics: {json.dumps(final_metrics, indent=2)}") except Exception as e: logger.error(f"Error during cleanup: {e}") raise finally: logger.info("WebSocket cleanup completed") async def _clear_queues(self): """Clear all message queues""" try: logger.debug("Clearing all message queues...") queues = [self.mempool_queue, self.price_queue, self.process_queue] for queue in queues: while not queue.empty(): try: queue.get_nowait() except asyncio.QueueEmpty: break logger.info("All queues cleared") except Exception as e: logger.error(f"Error clearing queues: {e}") async def get_current_price(self) -> Optional[PriceData]: """Get current price data with freshness check""" if self.price_data and time.time() - self.last_price_update < 5: logger.debug(f"[DEBUG] Current price data retrieved: {self.price_data}") return self.price_data logger.debug("[DEBUG] No fresh price data available.") return None async def get_connection_status(self) -> Dict: """Get current connection status""" status = { 'helius_connected': bool(self.ws_helius and not self.ws_helius.closed), 'quicknode_connected': bool(self.ws_quicknode and not self.ws_quicknode.closed), 'price_data_fresh': bool(self.price_data and time.time() - self.last_price_update < 5), 'mempool_queue_size': self.mempool_queue.qsize(), 'price_queue_size': self.price_queue.qsize(), 'process_queue_size': self.process_queue.qsize(), 'metrics': { 'messages_received': self.metrics.messages_received, 'reconnect_count': self.metrics.reconnect_count, 'errors': self.metrics.errors, 'mempool_hits': self.metrics.mempool_hits, 'processed_txs': self.metrics.processed_txs, 'uptime': time.time() - self.metrics.start_time } } logger.debug(f"[DEBUG] Connection status: {status}") return status async def get_market_depth(self) -> Dict: """Get current market depth""" logger.debug("[INFO] Getting current market depth...") try: if not self.ws_quicknode: logger.warning("[WARNING] QuickNode WebSocket is not connected.") return {} await self.ws_quicknode.send(json.dumps({ "jsonrpc": "2.0", "id": "depth", "method": "getOrderbook", "params": [ self.SOL_WSOL_MARKET, 20 # Depth levels ] })) response = await self.ws_quicknode.recv() data = json.loads(response) if 'result' in data: logger.debug("[DEBUG] Market depth data retrieved.") return { 'bids': data['result']['bids'], 'asks': data['result']['asks'], 'timestamp': time.time() } logger.warning("[WARNING] No market depth data found in response.") return {} except Exception as e: logger.error(f"Error getting market depth: {e}") return {} def _calculate_latency(self, start_time: float) -> float: """Calculate message processing latency""" latency = time.time() - start_time self.metrics.total_latency += latency self.metrics.peak_latency = max(self.metrics.peak_latency, latency) logger.debug(f"[DEBUG] Latency calculated: {latency:.3f}s") return latency def _save_statistics(self): """Save MEV protection statistics""" try: logger.info(f"MEV Protection Stats: {json.dumps(self._stats, indent=2)}") except Exception as e: logger.error(f"Error saving statistics: {e}") def _save_metrics(self): """Save trading metrics""" try: metrics = { 'price_updates': len(self.price_queue._queue), 'mempool_updates': len(self.mempool_queue._queue), 'last_price': self.price_data.price if self.price_data else None, 'last_update': self.last_price_update } logger.info(f"Trading Metrics: {json.dumps(metrics, indent=2)}") except Exception as e: logger.error(f"Error saving metrics: {e}") def _log_error(self, error: Exception, context: str = ""): """Log error with context""" error_msg = f"{context} error: {str(error)}" if context else str(error) logger.error(error_msg) self.metrics.errors += 1 # Example usage: if __name__ == "__main__": client = HeliusWebSocketClient() try: asyncio.run(client.start()) except Exception as e: logger.error(f"Error running WebSocket Client: {e}") finally: asyncio.run(client.cleanup())Key Features
-
Initialization: Initializes multiple queues (
mempool_queue,price_queue, andprocess_queue) for handling different types of messages collected via WebSockets. -
Queue Clearing: The
_clear_queuesmethod clears all message queues, ensuring that resources are managed effectively when needed. -
Maintaining Connections: The methods
start,_maintain_helius_connection, and_maintain_quicknode_connectionhandle the connection lifecycle and reconnection logic. -
Message Processing: The
_process_messagesand_process_message_batchmethods are responsible for handling messages from the various queues, processing them in batches for efficiency. -
Health Monitoring: Includes
_monitor_connectionsto check the status of WebSocket connections and the freshness of price data. -
Cleanup Logic: The
cleanupmethod ensures that resources are appropriately released, such as closing WebSocket connections and clearing queues before shutting down. -
Error Handling: Enhanced error handling through context-aware logging to capture specific points of failure.
Feel free to adapt or expand this code based on your specific requirements or trading strategy logic! If you need further modifications or additional features, let me know!
-
- User
In mev_protection.py
class EnhancedMevProtection: def init(self): self._stats = { 'transactions_analyzed': 0, 'opportunities_detected': 0, 'successful_executions': 0, 'failed_executions': 0, 'total_profit': 0.0, 'average_execution_time': 0.0, 'sandwich_attempts': 0, 'sandwich_successes': 0, }
def _save_statistics(self): """Save MEV protection statistics""" try: logger.info(f"MEV Protection Stats: {json.dumps(self._stats, indent=2)}") except Exception as e: logger.error(f"Error saving statistics: {e}") async def cleanup(self): try: self._save_statistics() logger.info("MEV protection cleanup completed") except Exception as e: logger.error(f"MEV protection cleanup error: {e}") <---- goes into --->>>>import osfrom dataclasses import dataclass from typing import Optional import logging import base64 from decimal import Decimal import time from collections import deque from config_loader import config
Load environment variables
RPC_URL = os.getenv("RPC_URL") WS_URL = os.getenv("WS_URL") HELIUS_API_KEY = os.getenv("HELIUS_API_KEY") MIN_PRIORITY_FEE = int(os.getenv("PRIORITY_FEE")) MAX_PRIORITY_FEE = int(os.getenv("MAX_PRIORITY_FEE")) COMPUTE_LIMIT = int(os.getenv("COMPUTE_LIMIT")) SLIPPAGE = float(os.getenv("SANDWICH_MAX_IMPACT")) BUNDLE_TIMEOUT = float(os.getenv("QUEUE_SIZE")) / 1000 # Assuming QUEUE_SIZE relates to seconds MAX_RETRIES = int(os.getenv("MAX_RETRIES")) # Placeholder for max retries LOGGING_LEVEL = os.getenv("LOG_LEVEL").upper()
Configure logging to only output to the console
logging.basicConfig( level=getattr(logging, LOGGING_LEVEL, logging.DEBUG), format='%(asctime)s.%(msecs)03d [%(levelname)s] [%(filename)s:%(lineno)d] %(message)s', datefmt='%Y-%m-%d %H:%M:%S', handlers=[logging.StreamHandler()] # Only log to the console )
logger = logging.getLogger(name)
@dataclass class MevConfig: max_fee_multiplier: float = 1.5 # Match config fee_multiplier min_confirmations: int = 1 max_retries: int = MAX_RETRIES # Replace with MAX_RETRIES from .env backoff_time: float = 0.2 min_priority_fee: int = MIN_PRIORITY_FEE # Match config min_priority_fee max_priority_fee: int = MAX_PRIORITY_FEE # Match config max_priority_fee bundle_timeout: float = BUNDLE_TIMEOUT # Retrieve from .env max_bundle_size: int = 3 compute_budget: int = COMPUTE_LIMIT # Match config compute_limit slippage: float = SLIPPAGE # Retrieve slippage from .env
class EnhancedMevProtection: def init(self, config: MevConfig): self.config = config self.pending_bundles = {} self.last_execution = time.time()
# Enhanced network monitoring self.network_stats = { 'recent_blocks': deque(maxlen=100), 'block_times': deque(maxlen=50), 'congestion_levels': deque(maxlen=20), 'latency_measurements': deque(maxlen=100), 'priority_fees': deque(maxlen=50) } # Bundle management self.active_bundles = {} self.bundle_history = deque(maxlen=1000) self.priority_thresholds = { 'low': 50000, # Placeholder for constants 'medium': 200000, # Placeholder for constants 'high': 500000, # Placeholder for constants 'urgent': self.config.max_priority_fee # Use max priority fee from config } logger.info("Enhanced MEV Protection initialized") async def protect_transaction(self, tx_data: dict) -> Optional[dict]: """Protect transaction with advanced bundling and fee optimization""" try: if not self._can_execute(): logger.info("Rate limiting in effect") return None # Calculate optimal priority fee based on network conditions priority_fee = await self._calculate_optimal_fee(tx_data) # Create atomic bundle bundle = await self._create_atomic_bundle( tx_data, priority_fee ) if not bundle: return None # Add advanced protection parameters protected_tx = { **bundle, 'minConfirmations': self._get_required_confirmations(), 'maxPriorityFee': priority_fee, 'computeUnits': self._calculate_compute_units(tx_data), 'bundleProtection': True, 'backrunProtection': self._should_add_backrun_protection(), 'maxRetries': self.config.max_retries } self.last_execution = time.time() return protected_tx except Exception as e: logger.error(f"Transaction protection error: {e}") return None async def _calculate_optimal_fee(self, tx_data: dict) -> int: """Calculate optimal priority fee based on multiple factors""" try: base_fee = self.config.min_priority_fee # Network congestion adjustment congestion_multiplier = self._get_congestion_multiplier() base_fee *= congestion_multiplier # Profit-based adjustment if 'expectedProfit' in tx_data: profit_multiplier = min( tx_data['expectedProfit'] / 0.001, # 0.1% baseline 2.0 # Cap at 2x ) base_fee *= profit_multiplier # Competition-based adjustment if self.network_stats['priority_fees']: avg_fee = sum(self.network_stats['priority_fees']) / len(self.network_stats['priority_fees']) if base_fee < avg_fee: base_fee = (base_fee + avg_fee) / 2 # Ensure within limits final_fee = min( max(int(base_fee), self.config.min_priority_fee), self.config.max_priority_fee ) logger.info(f"Calculated priority fee: {final_fee}") return final_fee except Exception as e: logger.error(f"Fee calculation error: {e}") return self.config.min_priority_fee async def _create_atomic_bundle( self, tx_data: dict, priority_fee: int ) -> Optional[dict]: """Create atomic transaction bundle with backrun protection""" try: # Generate bundle ID bundle_id = self._generate_bundle_id(tx_data) # Check if similar bundle already exists if bundle_id in self.active_bundles: logger.warning(f"Similar bundle {bundle_id} already active") return None # Create atomic bundle bundle = { 'id': bundle_id, 'transactions': [ self._create_compute_budget_ix(priority_fee), self._create_main_transaction(tx_data), ], 'timeout': time.time() + self.config.bundle_timeout } # Add backrun protection if needed if self._should_add_backrun_protection(): backrun_tx = self._create_backrun_transaction(tx_data) if backrun_tx: bundle['transactions'].append(backrun_tx) # Store bundle self.active_bundles[bundle_id] = bundle return bundle except Exception as e: logger.error(f"Bundle creation error: {e}") return None def _should_add_backrun_protection(self) -> bool: """Determine if backrun protection is needed""" try: # Check network conditions high_congestion = self.network_stats['congestion_levels'][-1] > 0.8 recent_backruns = sum(1 for bundle in self.bundle_history if bundle.get('backrunDetected', False)) return high_congestion or recent_backruns > 5 except Exception as e: logger.error(f"Backrun protection check error: {e}") return False def update_network_stats(self, block_time: float, latency: float, priority_fee: int ): """Update network statistics with enhanced monitoring""" try: # Update statistics self.network_stats['block_times'].append(block_time) self.network_stats['latency_measurements'].append(latency) self.network_stats['priority_fees'].append(priority_fee) # Calculate congestion avg_block_time = sum(self.network_stats['block_times']) / len(self.network_stats['block_times']) congestion = min(1.0, avg_block_time / 0.4) # 0.4s target block time self.network_stats['congestion_levels'].append(congestion) logger.info(f"Network stats updated - Congestion: {congestion:.2f}, Latency: {latency:.2f}ms") except Exception as e: logger.error(f"Network stats update error: {e}") def cleanup(self): """Clean up resources and save statistics""" try: # Clear active bundles self.active_bundles.clear() # Save statistics for analysis self._save_statistics() # Clear memory self.network_stats = {k: deque(maxlen=v.maxlen) for k, v in self.network_stats.items()} logger.info("MEV protection cleanup completed") except Exception as e: logger.error(f"Cleanup error: {e}") raise - Assistant
Here’s the complete
mev_protection.pyscript, integrating the new features and modifications regarding theEnhancedMevProtectionclass, including saving statistics and cleaning up.import os from dataclasses import dataclass from typing import Optional import logging import base64 from decimal import Decimal import time from collections import deque from config_loader import config # Load environment variables RPC_URL = os.getenv("RPC_URL") WS_URL = os.getenv("WS_URL") HELIUS_API_KEY = os.getenv("HELIUS_API_KEY") MIN_PRIORITY_FEE = int(os.getenv("PRIORITY_FEE")) MAX_PRIORITY_FEE = int(os.getenv("MAX_PRIORITY_FEE")) COMPUTE_LIMIT = int(os.getenv("COMPUTE_LIMIT")) SLIPPAGE = float(os.getenv("SANDWICH_MAX_IMPACT")) BUNDLE_TIMEOUT = float(os.getenv("QUEUE_SIZE")) / 1000 # Assuming QUEUE_SIZE relates to seconds MAX_RETRIES = int(os.getenv("MAX_RETRIES")) # Placeholder for max retries LOGGING_LEVEL = os.getenv("LOG_LEVEL").upper() # Configure logging to only output to the console logging.basicConfig( level=getattr(logging, LOGGING_LEVEL, logging.DEBUG), format='%(asctime)s.%(msecs)03d [%(levelname)s] [%(filename)s:%(lineno)d] %(message)s', datefmt='%Y-%m-%d %H:%M:%S', handlers=[logging.StreamHandler()] # Only log to the console ) logger = logging.getLogger(__name__) @dataclass class MevConfig: max_fee_multiplier: float = 1.5 min_confirmations: int = 1 max_retries: int = MAX_RETRIES backoff_time: float = 0.2 min_priority_fee: int = MIN_PRIORITY_FEE max_priority_fee: int = MAX_PRIORITY_FEE bundle_timeout: float = BUNDLE_TIMEOUT max_bundle_size: int = 3 compute_budget: int = COMPUTE_LIMIT slippage: float = SLIPPAGE class EnhancedMevProtection: def __init__(self, config: MevConfig): self.config = config self.pending_bundles = {} self.last_execution = time.time() # Enhanced network monitoring self.network_stats = { 'recent_blocks': deque(maxlen=100), 'block_times': deque(maxlen=50), 'congestion_levels': deque(maxlen=20), 'latency_measurements': deque(maxlen=100), 'priority_fees': deque(maxlen=50) } # Bundle management self.active_bundles = {} self.bundle_history = deque(maxlen=1000) self.priority_thresholds = { 'low': 50000, 'medium': 200000, 'high': 500000, 'urgent': self.config.max_priority_fee } # Initialize statistics self._stats = { 'transactions_analyzed': 0, 'opportunities_detected': 0, 'successful_executions': 0, 'failed_executions': 0, 'total_profit': 0.0, 'average_execution_time': 0.0, 'sandwich_attempts': 0, 'sandwich_successes': 0, } logger.info("Enhanced MEV Protection initialized") async def protect_transaction(self, tx_data: dict) -> Optional[dict]: """Protect transaction with advanced bundling and fee optimization""" try: if not self._can_execute(): logger.info("Rate limiting in effect") return None # Calculate optimal priority fee based on network conditions priority_fee = await self._calculate_optimal_fee(tx_data) # Create atomic bundle bundle = await self._create_atomic_bundle( tx_data, priority_fee ) if not bundle: return None # Add advanced protection parameters protected_tx = { **bundle, 'minConfirmations': self._get_required_confirmations(), 'maxPriorityFee': priority_fee, 'computeUnits': self._calculate_compute_units(tx_data), 'bundleProtection': True, 'backrunProtection': self._should_add_backrun_protection(), 'maxRetries': self.config.max_retries } self.last_execution = time.time() return protected_tx except Exception as e: logger.error(f"Transaction protection error: {e}") return None async def _calculate_optimal_fee(self, tx_data: dict) -> int: """Calculate optimal priority fee based on multiple factors""" try: base_fee = self.config.min_priority_fee # Network congestion adjustment congestion_multiplier = self._get_congestion_multiplier() base_fee *= congestion_multiplier # Profit-based adjustment if 'expectedProfit' in tx_data: profit_multiplier = min( tx_data['expectedProfit'] / 0.001, 2.0 ) base_fee *= profit_multiplier # Competition-based adjustment if self.network_stats['priority_fees']: avg_fee = sum(self.network_stats['priority_fees']) / len(self.network_stats['priority_fees']) if base_fee < avg_fee: base_fee = (base_fee + avg_fee) / 2 # Ensure within limits final_fee = min( max(int(base_fee), self.config.min_priority_fee), self.config.max_priority_fee ) logger.info(f"Calculated priority fee: {final_fee}") return final_fee except Exception as e: logger.error(f"Fee calculation error: {e}") return self.config.min_priority_fee async def _create_atomic_bundle( self, tx_data: dict, priority_fee: int ) -> Optional[dict]: """Create atomic transaction bundle with backrun protection""" try: # Generate bundle ID bundle_id = self._generate_bundle_id(tx_data) # Check if similar bundle already exists if bundle_id in self.active_bundles: logger.warning(f"Similar bundle {bundle_id} already active") return None # Create atomic bundle bundle = { 'id': bundle_id, 'transactions': [ self._create_compute_budget_ix(priority_fee), self._create_main_transaction(tx_data), ], 'timeout': time.time() + self.config.bundle_timeout } # Add backrun protection if needed if self._should_add_backrun_protection(): backrun_tx = self._create_backrun_transaction(tx_data) if backrun_tx: bundle['transactions'].append(backrun_tx) # Store bundle self.active_bundles[bundle_id] = bundle return bundle except Exception as e: logger.error(f"Bundle creation error: {e}") return None def _should_add_backrun_protection(self) -> bool: """Determine if backrun protection is needed""" try: # Check network conditions high_congestion = self.network_stats['congestion_levels'][-1] > 0.8 recent_backruns = sum(1 for bundle in self.bundle_history if bundle.get('backrunDetected', False)) return high_congestion or recent_backruns > 5 except Exception as e: logger.error(f"Backrun protection check error: {e}") return False def update_network_stats(self, block_time: float, latency: float, priority_fee: int): """Update network statistics with enhanced monitoring""" try: # Update statistics self.network_stats['block_times'].append(block_time) self.network_stats['latency_measurements'].append(latency) self.network_stats['priority_fees'].append(priority_fee) # Calculate congestion avg_block_time = sum(self.network_stats['block_times']) / len(self.network_stats['block_times']) congestion = min(1.0, avg_block_time / 0.4) # 0.4s target block time self.network_stats['congestion_levels'].append(congestion) logger.info(f"Network stats updated - Congestion: {congestion:.2f}, Latency: {latency:.2f}ms") except Exception as e: logger.error(f"Network stats update error: {e}") def _save_statistics(self): """Save MEV protection statistics""" try: logger.info(f"MEV Protection Stats: {json.dumps(self._stats, indent=2)}") except Exception as e: logger.error(f"Error saving statistics: {e}") async def cleanup(self): """Clean up resources and save statistics""" try: # Clear active bundles self.active_bundles.clear() # Save statistics for analysis self._save_statistics() # Clear memory self.network_stats = {k: deque(maxlen=v.maxlen) for k, v in self.network_stats.items()} logger.info("MEV protection cleanup completed") except Exception as e: logger.error(f"Cleanup error: {e}") raiseKey Features of the
EnhancedMevProtectionClass:-
Initialization: The
__init__method sets up various stats and network monitoring data structures when a new instance of the class is created. -
Transaction Protection: The
protect_transactionmethod manages the protection of transactions, calculating optimal fees and creating bundles for atomic transaction execution. -
Network Monitoring: Implements network statistics gathering to monitor congestion, latency, and priority fees, helping optimize transactional protection.
-
Cleanup Method: The
cleanupmethod is designed to clear active bundles, save statistics, and release any memory structures used during operations. -
Logging: Utilizes logging for error handling and reporting, allowing you to track the state and activity of the MEV protection system effectively.
-
Statistical Methods: Provided methods to save statistics that give insights into the performance of the MEV protection strategy.
Example Usage:
This implementation assumes that the requisite methods such as
_generate_bundle_id,_create_compute_budget_ix,_create_main_transaction, and_create_backrun_transactionare defined elsewhere in your codebase. You might need to integrate this class with the rest of your trading infrastructure accordingly.If you have more specific requirements or need adjustments, let me know!
-
- User
add this # In mempool_analyzer.py class EnhancedMempoolAnalyzer: def init(self): # Initialize CUDA streams if torch.cuda.is_available(): self.streams = { 'compute': torch.cuda.Stream(priority=-1), 'memory': torch.cuda.Stream(priority=-2), 'analysis': torch.cuda.Stream(priority=0) } else: self.streams = None
async def cleanup(self): try: if self.streams: # Synchronize and clean up CUDA streams for stream in self.streams.values(): stream.synchronize() torch.cuda.empty_cache() logger.info("Mempool analyzer cleanup complete") except Exception as e: logger.error(f"Mempool analyzer cleanup error: {e}") <----addd toooo ----> import asyncioimport logging import json import time import torch from typing import Dict, List, Optional from dataclasses import dataclass, field from collections import deque from decimal import Decimal import numpy as np import torch.amp as amp from concurrent.futures import ThreadPoolExecutor import os from config_loader import config
Load environment variables
MEMORY_FRACTION = float(os.getenv("GPU_MEMORY_FRACTION", 0.8)) SANDWICH_MIN_VICTIM_SIZE = float(os.getenv("SANDWICH_MIN_VICTIM_SIZE", 0.1)) SANDWICH_MIN_PROFIT = float(os.getenv("SANDWICH_MIN_PROFIT", 0.002)) IMPACT_MIN_THRESHOLD = float(os.getenv("IMPACT_MIN_THRESHOLD", 0.003)) IMPACT_SIZE_MULTIPLIER = float(os.getenv("IMPACT_SIZE_MULTIPLIER", 1.0))
Configure logging
logging.basicConfig( level=logging.DEBUG, format='%(asctime)s.%(msecs)03d [%(levelname)s] [%(filename)s:%(lineno)d] %(message)s', datefmt='%Y-%m-%d %H:%M:%S', handlers=[logging.StreamHandler()] ) logger = logging.getLogger(name)
@dataclass class MempoolMetrics: transactions_analyzed: int = 0 opportunities_found: int = 0 sandwich_attempts: int = 0 impact_attempts: int = 0 successful_execution: int = 0 avg_processing_time: float = 0.0 peak_memory: float = 0.0 total_profit: Decimal = Decimal('0.0') sandwich_profits: Decimal = Decimal('0.0') impact_profits: Decimal = Decimal('0.0') start_time: float = field(default_factory=time.time)
@dataclass class TransactionInfo: raw_data: bytes pool_address: str amount_in: float amount_out: float timestamp: float tx_type: str # 'sandwich_victim', 'impact', 'normal' priority: int metadata: Dict = field(default_factory=dict)
class EnhancedMempoolAnalyzer: def init(self, trading_core): # Initialize buffer size from environment variables self.buffer_size = int(os.getenv('VECTOR_BATCH_SIZE', '64')) self.feature_dim = int(os.getenv('VECTOR_DIMENSION', '512'))
self.trading_core = trading_core self._setup_buffers() def _setup_buffers(self): # First create on CPU then move to GPU self.tx_buffer = torch.zeros( (self.buffer_size, self.feature_dim), dtype=torch.float32, device='cpu' ).pin_memory() # Then move to GPU self.tx_buffer = self.tx_buffer.to('cuda:0') # Initialize impact buffer self.impact_buffer = torch.zeros( self.buffer_size, dtype=torch.float32, device='cpu' ).pin_memory().to('cuda:0') def _setup_buffers(self): """Initialize GPU buffers for transaction analysis""" try: # First create on CPU then move to GPU self.tx_buffer = torch.zeros( (self.buffer_size, self.feature_dim), dtype=torch.float32, device='cpu' # Create on CPU first ).pin_memory() # Pin the CPU memory # Then move to GPU self.tx_buffer = self.tx_buffer.to('cuda:0') # Specify GPU device index # Do the same for other buffers if you have them self.impact_buffer = torch.zeros( self.buffer_size, dtype=torch.float32, device='cpu' ).pin_memory().to('cuda:0') torch.cuda.synchronize() logger.debug("GPU buffers initialized") except Exception as e: logger.error(f"Buffer initialization error: {e}") raise async def analyze_transaction(self, tx_data: bytes) -> Optional[TransactionInfo]: """Analyze transaction for opportunities""" try: start_time = time.time() # Convert to tensor for GPU processing with torch.cuda.stream(self.streams['analyze']): tx_tensor = await self._prepare_transaction(tx_data) if tx_tensor is None: return None # Parallel opportunity analysis sandwich_result, impact_result = await asyncio.gather( self._analyze_sandwich_opportunity(tx_tensor, tx_data), self._analyze_impact_opportunity(tx_tensor, tx_data) ) # Select best opportunity result = self._select_best_opportunity(sandwich_result, impact_result) if result: self.metrics.opportunities_found += 1 # Update metrics self.metrics.transactions_analyzed += 1 process_time = time.time() - start_time self.metrics.avg_processing_time = ( self.metrics.avg_processing_time * 0.95 + process_time * 0.05 ) return result except Exception as e: logger.error(f"Transaction analysis error: {e}") return None async def _analyze_sandwich_opportunity( self, tx_tensor: torch.Tensor, raw_data: bytes ) -> Optional[TransactionInfo]: """Analyze for sandwich opportunities""" try: with torch.cuda.stream(self.streams['sandwich']): # Extract transaction data amount_in = float(tx_tensor[0]) amount_out = float(tx_tensor[1]) pool = tx_tensor[2:34].cpu().numpy().tobytes().hex() # Skip if amount too small if amount_in < SANDWICH_MIN_VICTIM_SIZE: return None # Calculate potential profit profit = await self._calculate_sandwich_profit( amount_in, amount_out, pool ) if profit < SANDWICH_MIN_PROFIT: return None self.metrics.sandwich_attempts += 1 return TransactionInfo( raw_data=raw_data, pool_address=pool, amount_in=amount_in, amount_out=amount_out, timestamp=time.time(), tx_type='sandwich_victim', priority=1, metadata={'potential_profit': profit} ) except Exception as e: logger.error(f"Sandwich analysis error: {e}") return None async def _analyze_impact_opportunity( self, tx_tensor: torch.Tensor, raw_data: bytes ) -> Optional[TransactionInfo]: """Analyze for price impact opportunities""" try: with torch.cuda.stream(self.streams['impact']): # Extract data amount_in = float(tx_tensor[0]) amount_out = float(tx_tensor[1]) pool = tx_tensor[2:34].cpu().numpy().tobytes().hex() # Calculate impact impact = await self._calculate_price_impact( amount_in, amount_out ) if impact < IMPACT_MIN_THRESHOLD: return None # Calculate optimal trade size trade_size = amount_in * IMPACT_SIZE_MULTIPLIER if trade_size < self.config['trading']['impact']['min_size']: return None self.metrics.impact_attempts += 1 return TransactionInfo( raw_data=raw_data, pool_address=pool, amount_in=trade_size, amount_out=0.0, # Will be calculated during execution timestamp=time.time(), tx_type='impact', priority=2, metadata={'impact': impact} ) except Exception as e: logger.error(f"Impact analysis error: {e}") return None @torch.amp.autocast('cuda') # Use torch's automatic mixed precision async def _calculate_price_impact(self, amount_in: float, amount_out: float) -> float: """Calculate price impact with GPU acceleration""" try: with torch.cuda.stream(self.streams['compute']): # Convert to tensors amount_in_tensor = torch.tensor(amount_in, device=self.device) amount_out_tensor = torch.tensor(amount_out, device=self.device) # Calculate impact price = amount_out_tensor / amount_in_tensor impact = torch.abs(1 - price) return float(impact) except Exception as e: logger.error(f"Impact calculation error: {e}") return float('inf') def _select_best_opportunity( self, sandwich: Optional[TransactionInfo], impact: Optional[TransactionInfo] ) -> Optional[TransactionInfo]: """Select best opportunity based on profit and priority""" try: opportunities = [x for x in [sandwich, impact] if x is not None] if not opportunities: return None # Sort by priority (lower is better) and profit return min( opportunities, key=lambda x: ( x.priority, -x.metadata.get('potential_profit', 0) ) ) except Exception as e: logger.error(f"Opportunity selection error: {e}") return None async def _prepare_transaction(self, tx_data: bytes) -> Optional[torch.Tensor]: """Prepare transaction data for GPU processing""" try: # Extract transaction data amount_in = int.from_bytes(tx_data[0:8], 'little') amount_out = int.from_bytes(tx_data[8:16], 'little') pool = tx_data[64:96] # Convert to tensor data = np.array( [amount_in, amount_out, *pool], dtype=np.float32 ) return torch.from_numpy(data).to( self.device, non_blocking=True ) except Exception as e: logger.error(f"Transaction preparation error: {e}") return None async def cleanup(self): """Cleanup resources properly""" try: # Clean GPU resources if torch.cuda.is_available(): torch.cuda.synchronize() for stream in self.streams.values(): stream.synchronize() self.tx_buffer = None self.pattern_buffer = None self.impact_buffer = None torch.cuda.empty_cache() # Clear data structures self.recent_txs.clear() self.victim_txs.clear() self.impact_txs.clear() self.sandwich_windows.clear() # Shutdown thread pool self.thread_pool.shutdown(wait=True) logger.info("Mempool analyzer cleanup complete") except Exception as e: logger.error(f"Cleanup error: {e}") raise - Assistant
Here's the complete
mempool_analyzer.pyscript, incorporating the new features you provided regarding theEnhancedMempoolAnalyzerclass with cleanup functionality. The class includes CUDA stream initialization and appropriate error handling.import asyncio import logging import json import time import torch from typing import Dict, List, Optional from dataclasses import dataclass, field from collections import deque from decimal import Decimal import numpy as np import torch.amp as amp from concurrent.futures import ThreadPoolExecutor import os from config_loader import config # Load environment variables MEMORY_FRACTION = float(os.getenv("GPU_MEMORY_FRACTION", 0.8)) SANDWICH_MIN_VICTIM_SIZE = float(os.getenv("SANDWICH_MIN_VICTIM_SIZE", 0.1)) SANDWICH_MIN_PROFIT = float(os.getenv("SANDWICH_MIN_PROFIT", 0.002)) IMPACT_MIN_THRESHOLD = float(os.getenv("IMPACT_MIN_THRESHOLD", 0.003)) IMPACT_SIZE_MULTIPLIER = float(os.getenv("IMPACT_SIZE_MULTIPLIER", 1.0)) # Configure logging logging.basicConfig( level=logging.DEBUG, format='%(asctime)s.%(msecs)03d [%(levelname)s] [%(filename)s:%(lineno)d] %(message)s', datefmt='%Y-%m-%d %H:%M:%S', handlers=[logging.StreamHandler()] # Only log to the console ) logger = logging.getLogger(__name__) @dataclass class MempoolMetrics: transactions_analyzed: int = 0 opportunities_found: int = 0 sandwich_attempts: int = 0 impact_attempts: int = 0 successful_execution: int = 0 avg_processing_time: float = 0.0 peak_memory: float = 0.0 total_profit: Decimal = Decimal('0.0') sandwich_profits: Decimal = Decimal('0.0') impact_profits: Decimal = Decimal('0.0') start_time: float = field(default_factory=time.time) @dataclass class TransactionInfo: raw_data: bytes pool_address: str amount_in: float amount_out: float timestamp: float tx_type: str # 'sandwich_victim', 'impact', 'normal' priority: int metadata: Dict = field(default_factory=dict) class EnhancedMempoolAnalyzer: def __init__(self, trading_core): # Initialize CUDA streams if torch.cuda.is_available(): self.streams = { 'compute': torch.cuda.Stream(priority=-1), 'memory': torch.cuda.Stream(priority=-2), 'analysis': torch.cuda.Stream(priority=0) } else: self.streams = None # Initialize buffer size from environment variables self.buffer_size = int(os.getenv('VECTOR_BATCH_SIZE', '64')) self.feature_dim = int(os.getenv('VECTOR_DIMENSION', '512')) self.trading_core = trading_core self.metrics = MempoolMetrics() self._setup_buffers() def _setup_buffers(self): """Initialize GPU buffers for transaction analysis""" try: # Create buffers on CPU first self.tx_buffer = torch.zeros( (self.buffer_size, self.feature_dim), dtype=torch.float32, device='cpu' # Create on CPU first ).pin_memory() # Pin the CPU memory # Move to GPU self.tx_buffer = self.tx_buffer.to('cuda:0') # Specify GPU device index # Initialize impact buffer self.impact_buffer = torch.zeros( self.buffer_size, dtype=torch.float32, device='cpu' ).pin_memory().to('cuda:0') torch.cuda.synchronize() logger.debug("GPU buffers initialized") except Exception as e: logger.error(f"Buffer initialization error: {e}") raise async def analyze_transaction(self, tx_data: bytes) -> Optional[TransactionInfo]: """Analyze transaction for opportunities""" try: start_time = time.time() # Convert to tensor for GPU processing with torch.cuda.stream(self.streams['analysis']): tx_tensor = await self._prepare_transaction(tx_data) if tx_tensor is None: return None # Parallel opportunity analysis sandwich_result, impact_result = await asyncio.gather( self._analyze_sandwich_opportunity(tx_tensor, tx_data), self._analyze_impact_opportunity(tx_tensor, tx_data) ) # Select best opportunity result = self._select_best_opportunity(sandwich_result, impact_result) if result: self.metrics.opportunities_found += 1 # Update metrics self.metrics.transactions_analyzed += 1 process_time = time.time() - start_time self.metrics.avg_processing_time = ( self.metrics.avg_processing_time * 0.95 + process_time * 0.05 ) return result except Exception as e: logger.error(f"Transaction analysis error: {e}") return None async def _analyze_sandwich_opportunity(self, tx_tensor: torch.Tensor, raw_data: bytes) -> Optional[TransactionInfo]: """Analyze for sandwich opportunities""" try: with torch.cuda.stream(self.streams['compute']): # Extract transaction data amount_in = float(tx_tensor[0]) amount_out = float(tx_tensor[1]) pool = tx_tensor[2:34].cpu().numpy().tobytes().hex() # Skip if amount too small if amount_in < SANDWICH_MIN_VICTIM_SIZE: return None # Calculate potential profit profit = await self._calculate_sandwich_profit(amount_in, amount_out, pool) if profit < SANDWICH_MIN_PROFIT: return None self.metrics.sandwich_attempts += 1 return TransactionInfo( raw_data=raw_data, pool_address=pool, amount_in=amount_in, amount_out=amount_out, timestamp=time.time(), tx_type='sandwich_victim', priority=1, metadata={'potential_profit': profit} ) except Exception as e: logger.error(f"Sandwich analysis error: {e}") return None async def _analyze_impact_opportunity(self, tx_tensor: torch.Tensor, raw_data: bytes) -> Optional[TransactionInfo]: """Analyze for price impact opportunities""" try: with torch.cuda.stream(self.streams['compute']): # Extract data amount_in = float(tx_tensor[0]) amount_out = float(tx_tensor[1]) pool = tx_tensor[2:34].cpu().numpy().tobytes().hex() # Calculate impact impact = await self._calculate_price_impact(amount_in, amount_out) if impact < IMPACT_MIN_THRESHOLD: return None # Calculate optimal trade size trade_size = amount_in * IMPACT_SIZE_MULTIPLIER if trade_size < self.config['trading']['impact']['min_size']: return None self.metrics.impact_attempts += 1 return TransactionInfo( raw_data=raw_data, pool_address=pool, amount_in=trade_size, amount_out=0.0, # Will be calculated during execution timestamp=time.time(), tx_type='impact', priority=2, metadata={'impact': impact} ) except Exception as e: logger.error(f"Impact analysis error: {e}") return None @torch.amp.autocast('cuda') # Use torch's automatic mixed precision async def _calculate_price_impact(self, amount_in: float, amount_out: float) -> float: """Calculate price impact with GPU acceleration""" try: with torch.cuda.stream(self.streams['compute']): # Convert to tensors amount_in_tensor = torch.tensor(amount_in, device=self.device) amount_out_tensor = torch.tensor(amount_out, device=self.device) # Calculate impact price = amount_out_tensor / amount_in_tensor impact = torch.abs(1 - price) return float(impact) except Exception as e: logger.error(f"Impact calculation error: {e}") return float('inf') def _select_best_opportunity(self, sandwich: Optional[TransactionInfo], impact: Optional[TransactionInfo]) -> Optional[TransactionInfo]: """Select best opportunity based on profit and priority""" try: opportunities = [x for x in [sandwich, impact] if x is not None] if not opportunities: return None # Sort by priority (lower is better) and profit return min( opportunities, key=lambda x: ( x.priority, -x.metadata.get('potential_profit', 0) ) ) except Exception as e: logger.error(f"Opportunity selection error: {e}") return None async def _prepare_transaction(self, tx_data: bytes) -> Optional[torch.Tensor]: """Prepare transaction data for GPU processing""" try: # Extract transaction data amount_in = int.from_bytes(tx_data[0:8], 'little') amount_out = int.from_bytes(tx_data[8:16], 'little') pool = tx_data[64:96] # Convert to tensor data = np.array( [amount_in, amount_out, *pool], dtype=np.float32 ) return torch.from_numpy(data).to( self.device, non_blocking=True ) except Exception as e: logger.error(f"Transaction preparation error: {e}") return None async def cleanup(self): """Cleanup resources properly""" try: if self.streams: # Synchronize and clean up CUDA streams for stream in self.streams.values(): stream.synchronize() torch.cuda.empty_cache() logger.info("Mempool analyzer cleanup complete") except Exception as e: logger.error(f"Mempool analyzer cleanup error: {e}") raiseKey Features of the
EnhancedMempoolAnalyzerClass:-
CUDA Stream Initialization: Streams are created only if CUDA is available to leverage GPU acceleration for computation.
-
Buffer Setup: The
_setup_buffersmethod initializes tensors on the CPU before moving them to the GPU to facilitate fast processing and analysis of incoming transaction data. -
Transaction Analysis: The
analyze_transactionmethod manages the entire pipeline for analyzing transactions for sandwich and price impact opportunities. -
Opportunity Analysis: The
_analyze_sandwich_opportunityand_analyze_impact_opportunitymethods assess each transaction type and return relevantTransactionInfo. -
Cleanup Functionality: The
cleanupmethod ensures that CUDA streams are synchronized and memory is released efficiently. -
Logging and Metrics: Uses structured logging to capture errors and operational statuses and maintains a
MempoolMetricsinstance to keep track of various operational statistics. -
Error Handling: Each method encapsulates error handling to ensure graceful error logging and prevents crashes.
Feel free to adapt this code based on your system’s structure or any specific business logic required for your trading platform! If you have further adjustments or detailed functionality you want to implement, let me know!
-
- User
In solana_cli.py
class SolanaCLI: def init(self): self.monitor_task = None self._running = False
async def start_monitoring(self): self._running = True self.monitor_task = asyncio.create_task(self._monitor_loop()) async def cleanup(self): try: self._running = False if self.monitor_task: self.monitor_task.cancel() try: await self.monitor_task except asyncio.CancelledError: pass logger.info("SolanaCLI cleanup completed") except Exception as e: logger.error(f"SolanaCLI cleanup error: {e}") <----addd to ---->>>>import asyncioimport logging import time import torch import os from enum import Enum from pathlib import Path from concurrent.futures import ThreadPoolExecutor from typing import Optional, Tuple, Dict from dataclasses import dataclass from dotenv import load_dotenv from config_loader import config
Load environment variables
load_dotenv()
Configure logging
logging.basicConfig( level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s', handlers=[ logging.StreamHandler(), logging.FileHandler('solana_cli.log') ] ) logger = logging.getLogger(name)
class WSOLState(Enum): IDLE = "idle" UNWRAPPING = "unwrapping" WRAPPING = "wrapping" VERIFYING = "verifying" ERROR = "error"
@dataclass class TokenBalances: sol: float = 0.0 wsol: float = 0.0 timestamp: float = 0.0
@dataclass class BalanceCacheEntry: balance: float timestamp: float dirty: bool = False
class SolanaCLI: def init(self, trading_core): """Initialize SolanaCLI with proper configuration""" self.trading_core = trading_core self.state = WSOLState.IDLE self.last_operation_time = time.time()
# Load environment variables self.WSOL_TOKEN_ACCOUNT = os.getenv('WSOL_TOKEN_ACCOUNT') if not self.WSOL_TOKEN_ACCOUNT: raise ValueError("WSOL_TOKEN_ACCOUNT environment variable not set") self.WSOL_MINT = "So11111111111111111111111111111111111111112" # WSOL Mint Address # Configuration loaded from environment variables self.MIN_SOL_BALANCE = float(os.getenv("MIN_SOL_BALANCE", 0.06)) self.CRITICAL_SOL_BALANCE = float(os.getenv("CRITICAL_SOL_BALANCE", 0.05)) self.WRAP_AMOUNT = float(os.getenv("WRAP_AMOUNT", 0.7)) self.MAX_RETRIES = int(os.getenv("MAX_RETRIES", 3)) self.RETRY_DELAY = float(os.getenv("RETRY_DELAY", 2.0)) self.UNWRAP_WAIT_TIME = float(os.getenv("UNWRAP_WAIT_TIME", 80)) self.VALIDATION_WAIT_TIME = float(os.getenv("VALIDATION_WAIT_TIME", 15)) self.BALANCE_CACHE_TIME = float(os.getenv("BALANCE_CACHE_TIME", 30.0)) # File paths self.BASE_DIR = Path("C:/solana_rust_bot") self.KEY_FILE = self.BASE_DIR / "id.json" # Operation state self.trading_active = True self.last_balances: Optional[TokenBalances] = None self.operation_lock = asyncio.Lock() self.balance_cache: Dict[str, BalanceCacheEntry] = {} # Performance optimization self.thread_pool = ThreadPoolExecutor(max_workers=2) self.device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') # GPU configuration if torch.cuda.is_available(): torch.backends.cuda.matmul.allow_tf32 = True torch.backends.cudnn.benchmark = True self._verify_environment() logger.info(f"SolanaCLI initialized with WSOL account: {self.WSOL_TOKEN_ACCOUNT}") def _verify_environment(self): """Verify environment and key files""" if not self.KEY_FILE.exists(): raise FileNotFoundError(f"Keypair file not found at {self.KEY_FILE}") async def execute_command(self, cmd: str) -> Tuple[int, bytes, bytes]: """Execute Solana CLI command with enhanced error handling.""" try: process = await asyncio.subprocess.create_subprocess_shell( cmd, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE ) stdout, stderr = await process.communicate() return process.returncode, stdout, stderr except Exception as e: logger.error(f"Command execution error: {e}") return -1, b"", str(e).encode() async def get_token_balance(self, token_account: str) -> float: """Get token account balance with caching""" # Check cache first cache_entry = self.balance_cache.get(token_account) current_time = time.time() if cache_entry and not cache_entry.dirty: if current_time - cache_entry.timestamp < self.BALANCE_CACHE_TIME: return cache_entry.balance try: cmd = f'spl-token balance {token_account} --url https://marga-e56m84-fast-mainnet.helius-rpc.com' returncode, stdout, stderr = await self.execute_command(cmd) if returncode == 0: try: balance = float(stdout.decode().strip()) # Update cache self.balance_cache[token_account] = BalanceCacheEntry( balance=balance, timestamp=current_time ) return balance except ValueError: logger.error("Failed to parse token balance") return 0.0 return 0.0 except Exception as e: logger.error(f"Balance check error: {e}") return 0.0 async def get_balances(self, force_refresh: bool = False) -> TokenBalances: """Get SOL and WSOL balances with optimized caching""" current_time = time.time() if (not force_refresh and self.last_balances and current_time - self.last_balances.timestamp < self.BALANCE_CACHE_TIME): return self.last_balances try: sol_balance = self.trading_core.get_balance() wsol_balance = await self.get_token_balance(self.WSOL_TOKEN_ACCOUNT) self.last_balances = TokenBalances( sol=sol_balance, wsol=wsol_balance, timestamp=current_time ) return self.last_balances except Exception as e: logger.error(f"Balance check error: {e}") if self.last_balances: return self.last_balances return TokenBalances() async def wrap_sol(self, amount: Optional[float] = None) -> bool: """Wrap SOL to WSOL with enhanced validation.""" async with self.operation_lock: if self.state != WSOLState.IDLE: logger.warning(f"Cannot wrap, current state: {self.state}") return False amount = amount or self.WRAP_AMOUNT self.trading_active = False self.state = WSOLState.WRAPPING try: # Verify sufficient balance balances = await self.get_balances(force_refresh=True) if balances.sol < amount + self.MIN_SOL_BALANCE: logger.error(f"Insufficient SOL balance for wrap: {balances.sol}") return False # Execute wrap cmd = f'spl-token wrap {amount} --fee-payer "{self.KEY_FILE}" --url https://marga-e56m84-fast-mainnet.helius-rpc.com' for attempt in range(self.MAX_RETRIES): returncode, stdout, stderr = await self.execute_command(cmd) if returncode == 0: await asyncio.sleep(2) if await self._verify_wrap_success(amount): logger.info(f"Wrap successful for {amount} SOL") # Invalidate cache self._invalidate_balance_cache() return True logger.error(f"Wrap attempt {attempt + 1} failed: {stderr.decode()}") if attempt < self.MAX_RETRIES - 1: await asyncio.sleep(self.RETRY_DELAY * (attempt + 1)) return False except Exception as e: logger.error(f"Wrap operation failed: {e}") return False finally: self.trading_active = True self.state = WSOLState.IDLE async def unwrap_sol(self) -> bool: """Unwrap WSOL to SOL with enhanced validation.""" async with self.operation_lock: if self.state != WSOLState.IDLE: logger.warning(f"Cannot unwrap, current state: {self.state}") return False self.trading_active = False self.state = WSOLState.UNWRAPPING try: # Check WSOL balance wsol_balance = await self.get_token_balance(self.WSOL_TOKEN_ACCOUNT) if wsol_balance <= 0: logger.warning("No WSOL balance to unwrap") return False # Execute unwrap cmd = f'spl-token unwrap {self.WSOL_TOKEN_ACCOUNT} --fee-payer "{self.KEY_FILE}" --url https://marga-e56m84-fast-mainnet.helius.rpc.com' for attempt in range(self.MAX_RETRIES): returncode, stdout, stderr = await self.execute_command(cmd) if returncode == 0: await asyncio.sleep(2) if await self._verify_unwrap_success(): logger.info("Unwrap successful") # Invalidate cache self._invalidate_balance_cache() return True logger.error(f"Unwrap attempt {attempt + 1} failed: {stderr.decode()}") if attempt < self.MAX_RETRIES - 1: await asyncio.sleep(self.RETRY_DELAY * (attempt + 1)) return False except Exception as e: logger.error(f"Unwrap operation failed: {e}") return False finally: self.trading_active = True self.state = WSOLState.IDLE async def _verify_wrap_success(self, amount: float) -> bool: """Verify wrap operation success with enhanced checking.""" try: new_balances = await self.get_balances(force_refresh=True) old_balances = self.last_balances or TokenBalances() wsol_increased = new_balances.wsol > old_balances.wsol sol_decreased = old_balances.sol - new_balances.sol >= amount return wsol_increased and sol_decreased except Exception as e: logger.error(f"Wrap verification error: {e}") return False async def _verify_unwrap_success(self) -> bool: """Verify unwrap operation success with enhanced checking.""" try: new_balances = await self.get_balances(force_refresh=True) old_balances = self.last_balances or TokenBalances() wsol_decreased = new_balances.wsol < old_balances.wsol sol_increased = new_balances.sol > old_balances.sol return wsol_decreased and sol_increased except Exception as e: logger.error(f"Unwrap verification error: {e}") return False def _invalidate_balance_cache(self): """Mark all balance cache entries as dirty.""" for entry in self.balance_cache.values(): entry.dirty = True async def start_monitoring(self): """Start balance monitoring.""" self.monitoring = True self.monitor_task = asyncio.create_task(self.monitor_balances()) logger.info("Balance monitoring started") async def monitor_balances(self): """Monitor SOL and WSOL balances with enhanced error handling.""" logger.info("Starting balance monitoring...") while self.monitoring: try: balances = await self.get_balances(force_refresh=True) logger.info(f"Current balances - SOL: {balances.sol:.6f}, WSOL: {balances.wsol:.6f}") if balances.sol <= self.CRITICAL_SOL_BALANCE and balances.wsol > 0: logger.info(f"SOL balance ({balances.sol}) below threshold. Initiating unwrap...") self.trading_active = False await asyncio.sleep(self.UNWRAP_WAIT_TIME) await self.unwrap_sol() await asyncio.sleep(self.VALIDATION_WAIT_TIME) for _ in range(self.MAX_RETRIES): if await self._verify_unwrap_success(): break await asyncio.sleep(self.VALIDATION_WAIT_TIME) await self.wrap_sol(self.WRAP_AMOUNT) await asyncio.sleep(self.VALIDATION_WAIT_TIME) for _ in range(self.MAX_RETRIES): if await self._verify_wrap_success(self.WRAP_AMOUNT): break await asyncio.sleep(self.VALIDATION_WAIT_TIME) self.trading_active = True elif balances.sol >= self.WRAP_AMOUNT: logger.info(f"SOL balance ({balances.sol}) above threshold. Wrapping SOL...") await self.wrap_sol() await asyncio.sleep(30) except Exception as e: logger.error(f"Balance monitoring error: {e}") await asyncio.sleep(5) def is_trading_active(self) -> bool: """Check if trading is active.""" return self.trading_active and self.state == WSOLState.IDLE async def cleanup(self): """Cleanup resources.""" logger.info("Cleaning up SolanaCLI resources") try: self.monitoring = False if self.monitor_task: self.monitor_task.cancel() try: await self.monitor_task except asyncio.CancelledError: pass self.thread_pool.shutdown(wait=True) if torch.cuda.is_available(): torch.cuda.empty_cache() self.balance_cache.clear() self.last_balances = None logger.info("Cleanup completed") except Exception as e: logger.error(f"Cleanup error: {e}") raise - Assistant
Here's the complete
solana_cli.pyscript, integrating the newSolanaCLIclass features you provided with cleanup functionality. This class handles SOL and WSOL operations, monitors balances, and utilizes the CUDA capabilities for performance improvements:import asyncio import logging import time import torch import os from enum import Enum from pathlib import Path from concurrent.futures import ThreadPoolExecutor from typing import Optional, Tuple, Dict from dataclasses import dataclass from dotenv import load_dotenv from config_loader import config # Load environment variables load_dotenv() # Configure logging logging.basicConfig( level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s', handlers=[ logging.StreamHandler(), logging.FileHandler('solana_cli.log') ] ) logger = logging.getLogger(__name__) class WSOLState(Enum): IDLE = "idle" UNWRAPPING = "unwrapping" WRAPPING = "wrapping" VERIFYING = "verifying" ERROR = "error" @dataclass class TokenBalances: sol: float = 0.0 wsol: float = 0.0 timestamp: float = 0.0 @dataclass class BalanceCacheEntry: balance: float timestamp: float dirty: bool = False class SolanaCLI: def __init__(self, trading_core): """Initialize SolanaCLI with proper configuration""" self.trading_core = trading_core self.state = WSOLState.IDLE self.last_operation_time = time.time() # Load environment variables self.WSOL_TOKEN_ACCOUNT = os.getenv('WSOL_TOKEN_ACCOUNT') if not self.WSOL_TOKEN_ACCOUNT: raise ValueError("WSOL_TOKEN_ACCOUNT environment variable not set") self.WSOL_MINT = "So11111111111111111111111111111111111111112" # WSOL Mint Address # Configuration loaded from environment variables self.MIN_SOL_BALANCE = float(os.getenv("MIN_SOL_BALANCE", 0.06)) self.CRITICAL_SOL_BALANCE = float(os.getenv("CRITICAL_SOL_BALANCE", 0.05)) self.WRAP_AMOUNT = float(os.getenv("WRAP_AMOUNT", 0.7)) self.MAX_RETRIES = int(os.getenv("MAX_RETRIES", 3)) self.RETRY_DELAY = float(os.getenv("RETRY_DELAY", 2.0)) self.UNWRAP_WAIT_TIME = float(os.getenv("UNWRAP_WAIT_TIME", 80)) self.VALIDATION_WAIT_TIME = float(os.getenv("VALIDATION_WAIT_TIME", 15)) self.BALANCE_CACHE_TIME = float(os.getenv("BALANCE_CACHE_TIME", 30.0)) # File paths self.BASE_DIR = Path("C:/solana_rust_bot") self.KEY_FILE = self.BASE_DIR / "id.json" # Operation state self.trading_active = True self.last_balances: Optional[TokenBalances] = None self.operation_lock = asyncio.Lock() self.balance_cache: Dict[str, BalanceCacheEntry] = {} # Performance optimization self.thread_pool = ThreadPoolExecutor(max_workers=2) self.device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') # GPU configuration if torch.cuda.is_available(): torch.backends.cuda.matmul.allow_tf32 = True torch.backends.cudnn.benchmark = True self._verify_environment() logger.info(f"SolanaCLI initialized with WSOL account: {self.WSOL_TOKEN_ACCOUNT}") def _verify_environment(self): """Verify environment and key files""" if not self.KEY_FILE.exists(): raise FileNotFoundError(f"Keypair file not found at {self.KEY_FILE}") async def execute_command(self, cmd: str) -> Tuple[int, bytes, bytes]: """Execute Solana CLI command with enhanced error handling.""" try: process = await asyncio.subprocess.create_subprocess_shell( cmd, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE ) stdout, stderr = await process.communicate() return process.returncode, stdout, stderr except Exception as e: logger.error(f"Command execution error: {e}") return -1, b"", str(e).encode() async def get_token_balance(self, token_account: str) -> float: """Get token account balance with caching""" # Check cache first cache_entry = self.balance_cache.get(token_account) current_time = time.time() if cache_entry and not cache_entry.dirty: if current_time - cache_entry.timestamp < self.BALANCE_CACHE_TIME: return cache_entry.balance try: cmd = f'spl-token balance {token_account} --url https://marga-e56m84-fast-mainnet.helius-rpc.com' returncode, stdout, stderr = await self.execute_command(cmd) if returncode == 0: try: balance = float(stdout.decode().strip()) # Update cache self.balance_cache[token_account] = BalanceCacheEntry( balance=balance, timestamp=current_time ) return balance except ValueError: logger.error("Failed to parse token balance") return 0.0 return 0.0 except Exception as e: logger.error(f"Balance check error: {e}") return 0.0 async def get_balances(self, force_refresh: bool = False) -> TokenBalances: """Get SOL and WSOL balances with optimized caching""" current_time = time.time() if (not force_refresh and self.last_balances and current_time - self.last_balances.timestamp < self.BALANCE_CACHE_TIME): return self.last_balances try: sol_balance = self.trading_core.get_balance() wsol_balance = await self.get_token_balance(self.WSOL_TOKEN_ACCOUNT) self.last_balances = TokenBalances( sol=sol_balance, wsol=wsol_balance, timestamp=current_time ) return self.last_balances except Exception as e: logger.error(f"Balance check error: {e}") if self.last_balances: return self.last_balances return TokenBalances() async def wrap_sol(self, amount: Optional[float] = None) -> bool: """Wrap SOL to WSOL with enhanced validation.""" async with self.operation_lock: if self.state != WSOLState.IDLE: logger.warning(f"Cannot wrap, current state: {self.state}") return False amount = amount or self.WRAP_AMOUNT self.trading_active = False self.state = WSOLState.WRAPPING try: # Verify sufficient balance balances = await self.get_balances(force_refresh=True) if balances.sol < amount + self.MIN_SOL_BALANCE: logger.error(f"Insufficient SOL balance for wrap: {balances.sol}") return False # Execute wrap cmd = f'spl-token wrap {amount} --fee-payer "{self.KEY_FILE}" --url https://marga-e56m84-fast-mainnet.helius-rpc.com' for attempt in range(self.MAX_RETRIES): returncode, stdout, stderr = await self.execute_command(cmd) if returncode == 0: await asyncio.sleep(2) if await self._verify_wrap_success(amount): logger.info(f"Wrap successful for {amount} SOL") # Invalidate cache self._invalidate_balance_cache() return True logger.error(f"Wrap attempt {attempt + 1} failed: {stderr.decode()}") if attempt < self.MAX_RETRIES - 1: await asyncio.sleep(self.RETRY_DELAY * (attempt + 1)) return False except Exception as e: logger.error(f"Wrap operation failed: {e}") return False finally: self.trading_active = True self.state = WSOLState.IDLE async def unwrap_sol(self) -> bool: """Unwrap WSOL to SOL with enhanced validation.""" async with self.operation_lock: if self.state != WSOLState.IDLE: logger.warning(f"Cannot unwrap, current state: {self.state}") return False self.trading_active = False self.state = WSOLState.UNWRAPPING try: # Check WSOL balance wsol_balance = await self.get_token_balance(self.WSOL_TOKEN_ACCOUNT) if wsol_balance <= 0: logger.warning("No WSOL balance to unwrap") return False # Execute unwrap cmd = f'spl-token unwrap {self.WSOL_TOKEN_ACCOUNT} --fee-payer "{self.KEY_FILE}" --url https://marga-e56m84-fast-mainnet.helius-rpc.com' for attempt in range(self.MAX_RETRIES): returncode, stdout, stderr = await self.execute_command(cmd) if returncode == 0: await asyncio.sleep(2) if await self._verify_unwrap_success(): logger.info("Unwrap successful") # Invalidate cache self._invalidate_balance_cache() return True logger.error(f"Unwrap attempt {attempt + 1} failed: {stderr.decode()}") if attempt < self.MAX_RETRIES - 1: await asyncio.sleep(self.RETRY_DELAY * (attempt + 1)) return False except Exception as e: logger.error(f"Unwrap operation failed: {e}") return False finally: self.trading_active = True self.state = WSOLState.IDLE async def _verify_wrap_success(self, amount: float) -> bool: """Verify wrap operation success with enhanced checking.""" try: new_balances = await self.get_balances(force_refresh=True) old_balances = self.last_balances or TokenBalances() wsol_increased = new_balances.wsol > old_balances.wsol sol_decreased = old_balances.sol - new_balances.sol >= amount return wsol_increased and sol_decreased except Exception as e: logger.error(f"Wrap verification error: {e}") return False async def _verify_unwrap_success(self) -> bool: """Verify unwrap operation success with enhanced checking.""" try: new_balances = await self.get_balances(force_refresh=True) old_balances = self.last_balances or TokenBalances() wsol_decreased = new_balances.wsol < old_balances.wsol sol_increased = new_balances.sol > old_balances.sol return wsol_decreased and sol_increased except Exception as e: logger.error(f"Unwrap verification error: {e}") return False def _invalidate_balance_cache(self): """Mark all balance cache entries as dirty.""" for entry in self.balance_cache.values(): entry.dirty = True async def start_monitoring(self): """Start balance monitoring.""" self.monitoring = True self.monitor_task = asyncio.create_task(self.monitor_balances()) logger.info("Balance monitoring started") async def monitor_balances(self): """Monitor SOL and WSOL balances with enhanced error handling.""" logger.info("Starting balance monitoring...") while self.monitoring: try: balances = await self.get_balances(force_refresh=True) logger.info(f"Current balances - SOL: {balances.sol:.6f}, WSOL: {balances.wsol:.6f}") if balances.sol <= self.CRITICAL_SOL_BALANCE and balances.wsol > 0: logger.info(f"SOL balance ({balances.sol}) below threshold. Initiating unwrap...") self.trading_active = False await asyncio.sleep(self.UNWRAP_WAIT_TIME) await self.unwrap_sol() await asyncio.sleep(self.VALIDATION_WAIT_TIME) for _ in range(self.MAX_RETRIES): if await self._verify_unwrap_success(): break await asyncio.sleep(self.VALIDATION_WAIT_TIME) await self.wrap_sol(self.WRAP_AMOUNT) await asyncio.sleep(self.VALIDATION_WAIT_TIME) for _ in range(self.MAX_RETRIES): if await self._verify_wrap_success(self.WRAP_AMOUNT): break await asyncio.sleep(self.VALIDATION_WAIT_TIME) self.trading_active = True elif balances.sol >= self.WRAP_AMOUNT: logger.info(f"SOL balance ({balances.sol}) above threshold. Wrapping SOL...") await self.wrap_sol() await asyncio.sleep(30) except Exception as e: logger.error(f"Balance monitoring error: {e}") await asyncio.sleep(5) def is_trading_active(self) -> bool: """Check if trading is active.""" return self.trading_active and self.state == WSOLState.IDLE async def cleanup(self): """Cleanup resources.""" logger.info("Cleaning up SolanaCLI resources") try: self.monitoring = False if self.monitor_task: self.monitor_task.cancel() try: await self.monitor_task except asyncio.CancelledError: pass self.thread_pool.shutdown(wait=True) if torch.cuda.is_available(): torch.cuda.empty_cache() self.balance_cache.clear() self.last_balances = None logger.info("Cleanup completed") except Exception as e: logger.error(f"Cleanup error: {e}") raiseKey Features of the
SolanaCLIClass:-
Initialization: The
__init__method sets up the environment for interacting with the Solana blockchain, including loading configurations, environment variables, and initializing CUDA settings if available. -
Token Management: The class features methods for wrapping and unwrapping SOL to WSOL, ensuring sufficient balances and validating operations.
-
Balance Monitoring: The
start_monitoring()method triggers balance checks at regular intervals to manage WSOL and SOL amounts dynamically. -
Cleanup Functionality: The
cleanup()method ensures proper resource management by shutting down the monitoring task, clearing caches, and releasing GPU memory if applicable. -
Concurrent Execution: Leverages asynchronous calls to execute commands and monitor balances in a non-blocking manner, improving responsiveness and performance.
-
Error Handling: Robust logging and error management throughout the code to track operational issues and facilitate debugging.
Example Usage:
This implementation assumes that you have defined the
trading_coreobject that interacts with the trading mechanism of your choice. Make sure to adapt any trading logic according to your application's functionality and your specific requirements.If you have further modifications or additional features you want to include, feel free to ask!
-
- User
In vector_store.py
class EnhancedVectorStore: def init(self): self.pattern_buffer = [] self.pattern_file = "patterns.json"
async def cleanup(self): try: # Save patterns before cleanup if self.pattern_buffer: with open(self.pattern_file, 'w') as f: json.dump(self.pattern_buffer, f) logger.info(f"Patterns saved to {self.pattern_file}") logger.info("Vector store cleanup complete") except Exception as e: logger.error(f"Vector store cleanup error: {e}") <----add to ----- >>>>import asyncioimport logging import os from typing import Optional, Dict, List from dataclasses import dataclass import time import torch import json from collections import deque from dotenv import load_dotenv from config_loader import config
Load environment variables
load_dotenv()
Configure logging
logging.basicConfig( level=logging.DEBUG, format='%(asctime)s - %(levelname)s - %(message)s', handlers=[ logging.StreamHandler(), logging.FileHandler('vector_store.log') ] ) logger = logging.getLogger(name)
@dataclass class VectorConfig: batch_size: int = int(os.getenv("VECTOR_BATCH_SIZE", 64)) vector_dim: int = int(os.getenv("VECTOR_DIM", 128)) similarity_threshold: float = float(os.getenv("SIMILARITY_THRESHOLD", 0.65)) max_patterns: int = int(os.getenv("MAX_PATTERNS", 10000)) cache_size: int = int(os.getenv("CACHE_SIZE", 1000)) memory_fraction: float = float(os.getenv("MEMORY_FRACTION", 0.8))
@dataclass class PoolPattern: vector: torch.Tensor timestamp: float metadata: Dict confidence: float = 0.0
class EnhancedVectorStore: def init(self): """Initialize vector store with enhanced GPU management.""" logger.debug("Initializing vector store...") self.config = VectorConfig()
# Initialize device self.device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') logger.info(f"Using device: {self.device}") self.patterns: Dict[str, PoolPattern] = {} # Initialize pattern storage self.pattern_buffer = torch.zeros( (self.config.max_patterns, self.config.vector_dim), device=self.device, dtype=torch.float16 ) # Cache structures self.cache: Dict[str, torch.Tensor] = {} self.metadata: Dict[str, Dict] = {} self._setup_tensors() self._warmup() logger.info("Vector store initialized successfully") def _setup_tensors(self): """Initialize GPU tensors with memory optimization.""" logger.debug("Setting up tensors...") if torch.cuda.is_available(): self.search_matrix = torch.zeros( (self.config.batch_size, self.config.vector_dim), device=self.device, dtype=torch.float16 ) self.similarity_cache = torch.zeros( (self.config.cache_size,), device=self.device, dtype=torch.float32 ) logger.debug("Tensor initialization complete") def _warmup(self): """Warm up GPU calculations with mixed precision.""" logger.debug("Warming up GPU...") if not torch.cuda.is_available(): return try: dummy_data = torch.randn( (32, self.config.vector_dim), device=self.device, dtype=torch.float16 ) logger.debug("Running warmup calculations...") for _ in range(10): self._calculate_similarity(dummy_data, dummy_data) logger.debug("GPU warmup complete") except Exception as e: logger.error(f"Warmup error: {e}") raise def _calculate_similarity(self, vec1: torch.Tensor, vec2: torch.Tensor) -> torch.Tensor: """Calculate cosine similarity.""" logger.debug("Calculating similarity...") try: if vec1.dim() == 1: vec1 = vec1.unsqueeze(0) if vec2.dim() == 1: vec2 = vec2.unsqueeze(0) vec1_norm = torch.nn.functional.normalize(vec1, p=2, dim=1) vec2_norm = torch.nn.functional.normalize(vec2, p=2, dim=1) similarity = torch.mm(vec1_norm, vec2_norm.t()) logger.debug(f"Calculated similarity: {similarity}") return similarity except Exception as e: logger.error(f"Similarity calculation error: {e}") return torch.tensor(0.0, device=self.device) async def store_pattern(self, pool_id: str, pattern_vector: torch.Tensor, metadata: Dict): """Store trading pattern with metadata and memory optimization.""" logger.debug(f"Storing pattern for pool ID: {pool_id}") try: if pattern_vector.device != self.device: pattern_vector = pattern_vector.to(self.device, non_blocking=True) pattern_vector = pattern_vector.to(dtype=torch.float16) confidence = await self._calculate_pattern_confidence(pattern_vector) logger.debug(f"Pattern confidence: {confidence}") if confidence >= self.config.similarity_threshold: self.patterns[pool_id] = PoolPattern( vector=pattern_vector, timestamp=time.time(), metadata=metadata, confidence=float(confidence) ) self.cache[pool_id] = pattern_vector self.metadata[pool_id] = metadata await self._cleanup_old_patterns() logger.debug(f"Pattern stored for pool {pool_id} with confidence {confidence:.3f}") except Exception as e: logger.error(f"Pattern storage error: {e}") raise async def _calculate_pattern_confidence(self, vector: torch.Tensor) -> float: """Calculate pattern confidence with optimized CUDA handling.""" logger.debug("Calculating pattern confidence...") try: if not self.patterns: return 1.0 existing_vectors = torch.stack([p.vector for p in self.patterns.values()], dim=0) similarities = self._calculate_similarity(vector, existing_vectors) max_similarity = torch.max(similarities).item() confidence = 1.0 - max_similarity logger.debug(f"Max similarity: {max_similarity}, Confidence: {confidence}") return float(confidence) except Exception as e: logger.error(f"Confidence calculation error: {e}") return 0.0 async def search_similar_patterns(self, query_vector: torch.Tensor, k: int = 5, min_similarity: float = 0.65) -> List[Dict]: """Search for similar patterns with optimized batch processing.""" logger.debug("Searching for similar patterns...") try: if query_vector.device != self.device: query_vector = query_vector.to(self.device, non_blocking=True) if not self.patterns: logger.debug("No patterns available for searching.") return [] pattern_vectors = [pattern.vector for pattern in self.patterns.values()] similarities = self._calculate_similarity(query_vector, torch.stack(pattern_vectors)) results = [ { 'pool_id': pool_id, 'similarity': float(sim), 'metadata': pattern.metadata, 'timestamp': pattern.timestamp } for pool_id, (pattern, sim) in zip(self.patterns.keys(), zip(self.patterns.values(), similarities)) if sim >= min_similarity ] results.sort(key=lambda x: x['similarity'], reverse=True) logger.debug(f"Found {len(results)} similar patterns.") return results[:k] except Exception as e: logger.error(f"Pattern search error: {e}") return [] async def _cleanup_old_patterns(self): """Clean up old patterns with memory optimization.""" logger.debug("Cleaning up old patterns...") if len(self.patterns) > self.config.max_patterns: sorted_patterns = sorted(self.patterns.items(), key=lambda x: x[1].timestamp) num_to_remove = len(self.patterns) - self.config.max_patterns for pool_id, _ in sorted_patterns[:num_to_remove]: logger.debug(f"Removing pattern for pool ID: {pool_id}") if pool_id in self.patterns: del self.patterns[pool_id] if pool_id in self.cache: del self.cache[pool_id] if pool_id in self.metadata: del self.metadata[pool_id] logger.info(f"Cleaned up {num_to_remove} old patterns") async def save_patterns(self, filepath: str = "patterns.json"): """Save patterns to disk with error handling.""" logger.debug("Saving patterns to disk...") try: save_data = {} for pool_id, pattern in self.patterns.items(): vector_cpu = pattern.vector.detach().cpu().numpy().tolist() save_data[pool_id] = { 'vector': vector_cpu, 'timestamp': pattern.timestamp, 'metadata': pattern.metadata, 'confidence': pattern.confidence } with open(filepath, 'w') as f: json.dump(save_data, f, indent=2) logger.info(f"Patterns saved to {filepath}") except Exception as e: logger.error(f"Pattern save error: {e}") raise async def load_patterns(self, filepath: str): """Load patterns from disk with optimized GPU transfer.""" logger.debug("Loading patterns from disk...") try: with open(filepath, 'r') as f: load_data = json.load(f) self.patterns.clear() self.cache.clear() self.metadata.clear() for pool_id, data in load_data.items(): vector = torch.tensor( data['vector'], dtype=torch.float16, device=self.device ) self.patterns[pool_id] = PoolPattern( vector=vector, timestamp=data['timestamp'], metadata=data['metadata'], confidence=data['confidence'] ) self.cache[pool_id] = vector self.metadata[pool_id] = data['metadata'] logger.info(f"Loaded {len(self.patterns)} patterns from {filepath}") except Exception as e: logger.error(f"Pattern load error: {e}") raise async def cleanup(self): """Clean up resources with proper CUDA memory management.""" logger.debug("Cleaning up vector store resources...") try: await self.save_patterns() if torch.cuda.is_available(): torch.cuda.synchronize() del self.pattern_buffer del self.search_matrix del self.similarity_cache torch.cuda.empty_cache() self.patterns.clear() self.cache.clear() self.metadata.clear() logger.info("Vector store cleanup complete") except Exception as e: logger.error(f"Cleanup error: {e}") raise - Assistant
Here’s the complete
vector_store.pyscript, including the new features for managing vectors, saving, and cleaning up properly, as you specified with theEnhancedVectorStoreclass. This version incorporates the requested cleanup functionality as well as enhanced logging and efficiency improvements.import asyncio import logging import os from typing import Optional, Dict, List from dataclasses import dataclass import time import torch import json from collections import deque from dotenv import load_dotenv from config_loader import config # Load environment variables load_dotenv() # Configure logging logging.basicConfig( level=logging.DEBUG, format='%(asctime)s - %(levelname)s - %(message)s', handlers=[ logging.StreamHandler(), logging.FileHandler('vector_store.log') ] ) logger = logging.getLogger(__name__) @dataclass class VectorConfig: batch_size: int = int(os.getenv("VECTOR_BATCH_SIZE", 64)) vector_dim: int = int(os.getenv("VECTOR_DIM", 128)) similarity_threshold: float = float(os.getenv("SIMILARITY_THRESHOLD", 0.65)) max_patterns: int = int(os.getenv("MAX_PATTERNS", 10000)) cache_size: int = int(os.getenv("CACHE_SIZE", 1000)) memory_fraction: float = float(os.getenv("MEMORY_FRACTION", 0.8)) @dataclass class PoolPattern: vector: torch.Tensor timestamp: float metadata: Dict confidence: float = 0.0 class EnhancedVectorStore: def __init__(self): """Initialize vector store with enhanced GPU management.""" logger.debug("Initializing vector store...") self.config = VectorConfig() # Initialize device self.device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') logger.info(f"Using device: {self.device}") self.patterns: Dict[str, PoolPattern] = {} # Initialize pattern storage self.pattern_buffer = torch.zeros( (self.config.max_patterns, self.config.vector_dim), device=self.device, dtype=torch.float16 ) # Cache structures self.cache: Dict[str, torch.Tensor] = {} self.metadata: Dict[str, Dict] = {} self._setup_tensors() self._warmup() logger.info("Vector store initialized successfully") def _setup_tensors(self): """Initialize GPU tensors with memory optimization.""" logger.debug("Setting up tensors...") if torch.cuda.is_available(): self.search_matrix = torch.zeros( (self.config.batch_size, self.config.vector_dim), device=self.device, dtype=torch.float16 ) self.similarity_cache = torch.zeros( (self.config.cache_size,), device=self.device, dtype=torch.float32 ) logger.debug("Tensor initialization complete") def _warmup(self): """Warm up GPU calculations with mixed precision.""" logger.debug("Warming up GPU...") if not torch.cuda.is_available(): return try: dummy_data = torch.randn( (32, self.config.vector_dim), device=self.device, dtype=torch.float16 ) logger.debug("Running warmup calculations...") for _ in range(10): self._calculate_similarity(dummy_data, dummy_data) logger.debug("GPU warmup complete") except Exception as e: logger.error(f"Warmup error: {e}") raise def _calculate_similarity(self, vec1: torch.Tensor, vec2: torch.Tensor) -> torch.Tensor: """Calculate cosine similarity.""" logger.debug("Calculating similarity...") try: if vec1.dim() == 1: vec1 = vec1.unsqueeze(0) if vec2.dim() == 1: vec2 = vec2.unsqueeze(0) vec1_norm = torch.nn.functional.normalize(vec1, p=2, dim=1) vec2_norm = torch.nn.functional.normalize(vec2, p=2, dim=1) similarity = torch.mm(vec1_norm, vec2_norm.t()) logger.debug(f"Calculated similarity: {similarity}") return similarity except Exception as e: logger.error(f"Similarity calculation error: {e}") return torch.tensor(0.0, device=self.device) async def store_pattern(self, pool_id: str, pattern_vector: torch.Tensor, metadata: Dict): """Store trading pattern with metadata and memory optimization.""" logger.debug(f"Storing pattern for pool ID: {pool_id}") try: if pattern_vector.device != self.device: pattern_vector = pattern_vector.to(self.device, non_blocking=True) pattern_vector = pattern_vector.to(dtype=torch.float16) confidence = await self._calculate_pattern_confidence(pattern_vector) logger.debug(f"Pattern confidence: {confidence}") if confidence >= self.config.similarity_threshold: self.patterns[pool_id] = PoolPattern( vector=pattern_vector, timestamp=time.time(), metadata=metadata, confidence=float(confidence) ) self.cache[pool_id] = pattern_vector self.metadata[pool_id] = metadata await self._cleanup_old_patterns() logger.debug(f"Pattern stored for pool {pool_id} with confidence {confidence:.3f}") except Exception as e: logger.error(f"Pattern storage error: {e}") raise async def _calculate_pattern_confidence(self, vector: torch.Tensor) -> float: """Calculate pattern confidence with optimized CUDA handling.""" logger.debug("Calculating pattern confidence...") try: if not self.patterns: return 1.0 existing_vectors = torch.stack([p.vector for p in self.patterns.values()], dim=0) similarities = self._calculate_similarity(vector, existing_vectors) max_similarity = torch.max(similarities).item() confidence = 1.0 - max_similarity logger.debug(f"Max similarity: {max_similarity}, Confidence: {confidence}") return float(confidence) except Exception as e: logger.error(f"Confidence calculation error: {e}") return 0.0 async def search_similar_patterns(self, query_vector: torch.Tensor, k: int = 5, min_similarity: float = 0.65) -> List[Dict]: """Search for similar patterns with optimized batch processing.""" logger.debug("Searching for similar patterns...") try: if query_vector.device != self.device: query_vector = query_vector.to(self.device, non_blocking=True) if not self.patterns: logger.debug("No patterns available for searching.") return [] pattern_vectors = [pattern.vector for pattern in self.patterns.values()] similarities = self._calculate_similarity(query_vector, torch.stack(pattern_vectors)) results = [ { 'pool_id': pool_id, 'similarity': float(sim), 'metadata': pattern.metadata, 'timestamp': pattern.timestamp } for pool_id, (pattern, sim) in zip(self.patterns.keys(), zip(self.patterns.values(), similarities)) if sim >= min_similarity ] results.sort(key=lambda x: x['similarity'], reverse=True) logger.debug(f"Found {len(results)} similar patterns.") return results[:k] except Exception as e: logger.error(f"Pattern search error: {e}") return [] async def _cleanup_old_patterns(self): """Clean up old patterns with memory optimization.""" logger.debug("Cleaning up old patterns...") if len(self.patterns) > self.config.max_patterns: sorted_patterns = sorted(self.patterns.items(), key=lambda x: x[1].timestamp) num_to_remove = len(self.patterns) - self.config.max_patterns for pool_id, _ in sorted_patterns[:num_to_remove]: logger.debug(f"Removing pattern for pool ID: {pool_id}") if pool_id in self.patterns: del self.patterns[pool_id] if pool_id in self.cache: del self.cache[pool_id] if pool_id in self.metadata: del self.metadata[pool_id] logger.info(f"Cleaned up {num_to_remove} old patterns") async def save_patterns(self, filepath: str = "patterns.json"): """Save patterns to disk with error handling.""" logger.debug("Saving patterns to disk...") try: save_data = {} for pool_id, pattern in self.patterns.items(): vector_cpu = pattern.vector.detach().cpu().numpy().tolist() save_data[pool_id] = { 'vector': vector_cpu, 'timestamp': pattern.timestamp, 'metadata': pattern.metadata, 'confidence': pattern.confidence } with open(filepath, 'w') as f: json.dump(save_data, f, indent=2) logger.info(f"Patterns saved to {filepath}") except Exception as e: logger.error(f"Pattern save error: {e}") raise async def load_patterns(self, filepath: str): """Load patterns from disk with optimized GPU transfer.""" logger.debug("Loading patterns from disk...") try: with open(filepath, 'r') as f: load_data = json.load(f) self.patterns.clear() self.cache.clear() self.metadata.clear() for pool_id, data in load_data.items(): vector = torch.tensor( data['vector'], dtype=torch.float16, device=self.device ) self.patterns[pool_id] = PoolPattern( vector=vector, timestamp=data['timestamp'], metadata=data['metadata'], confidence=data['confidence'] ) self.cache[pool_id] = vector self.metadata[pool_id] = data['metadata'] logger.info(f"Loaded {len(self.patterns)} patterns from {filepath}") except Exception as e: logger.error(f"Pattern load error: {e}") raise async def cleanup(self): """Clean up resources with proper CUDA memory management.""" logger.debug("Cleaning up vector store resources...") try: # Save patterns before cleanup if self.pattern_buffer: with open("patterns.json", 'w') as f: json.dump(self.pattern_buffer, f) logger.info("Patterns saved to patterns.json") if torch.cuda.is_available(): torch.cuda.synchronize() del self.pattern_buffer del self.search_matrix del self.similarity_cache torch.cuda.empty_cache() self.patterns.clear() self.cache.clear() self.metadata.clear() logger.info("Vector store cleanup complete") except Exception as e: logger.error(f"Cleanup error: {e}") raiseKey Features of the
EnhancedVectorStoreClass:-
Initialization: The constructor initializes the configuration for vector operations, allocates buffers in GPU memory, and sets up logging.
-
Pattern Management: Methods such as
store_pattern,search_similar_patterns, and_cleanup_old_patternsare used for adding vectors, querying similar patterns, and managing memory efficiently. -
Loading and Saving: The
load_patternsandsave_patternsmethods handle file I/O, storing and retrieving patterns from JSON files while ensuring the data is compatible with the GPU. -
Cleanup Functionality: The
cleanupmethod saves the current patterns before releasing resources and clearing buffers, ensuring no loss of data during shutdown. -
Error Handling: Robust error management is implemented throughout, logging errors and raising exceptions as necessary to prevent silent failures.
-
Batch Processing: Supports processing transactions in batches to leverage GPU capabilities efficiently, improving overall performance.
Feel free to modify or enhance this code based on your application requirements or if there are additional features you'd like to implement! If you have further questions or need assistance, just let me know!
-
- User
(quantum_beast) PS C:\solana_rust_bot> python "C:\solana_rust_bot\src\python\main.py" Traceback (most recent call last): File "C:\solana_rust_bot\src\python\main.py", line 23, in from mev_protection import EnhancedMevProtection, MevConfig File "C:\solana_rust_bot\src\python\mev_protection.py", line 21, in MAX_RETRIES = int(os.getenv("MAX_RETRIES")) # Placeholder for max retries ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ TypeError: int() argument must be a string, a bytes-like object or a real number, not 'NoneType' (quantum_beast) PS C:\solana_rust_bot> wtf brother lol
- Assistant
The error you're encountering is due to the
os.getenv("MAX_RETRIES")returningNone, which means that the environment variable"MAX_RETRIES"is not set. Theint()function cannot convertNoneTypeto an integer, which results in theTypeError.To fix this, you will need to ensure that the environment variable is defined properly in your environment or your
.envfile. Here are a few steps to troubleshoot and resolve this issue:-
Check Your .env File: Make sure that the
.envfile contains the definition forMAX_RETRIES. It should look something like this:MAX_RETRIES=3 -
Fallback Value: You can update your code to provide a default value if
os.getenv("MAX_RETRIES")returnsNone. Here’s how you can do that:MAX_RETRIES = int(os.getenv("MAX_RETRIES", 3)) # Default to 3 if not setThis way, if
MAX_RETRIESis not set, it will default to3. -
Ensure Environment Variables Are Loaded: Make sure that your environment variables are being loaded correctly when you run the script. You can add a debug statement to confirm:
logger.debug(f"MAX_RETRIES value: {os.getenv('MAX_RETRIES')}") -
Run the Code Again: After making these changes, try running your script again with:
python "C:\solana_rust_bot\src\python\main.py" -
Check for Other Variables: It's possible that other environment variables might also be missing or returning
None. Apply a similar pattern to others as needed.
Here’s the modified part of your code with a fallback value:
# In mev_protection.py MAX_RETRIES = int(os.getenv("MAX_RETRIES", 3)) # Provide a default value of 3Doing this will prevent your program from crashing due to missing environment variables, and you'll have control over what to set as defaults.
If you still encounter issues after these changes, let me know, and we can troubleshoot further!
-