Is there a nice way for me to be doing the data analysis on this and making pretty…
- User
Is there a nice way for me to be doing the data analysis on this and making pretty charts,m etc.:
#!/usr/bin/env python3 from collections import defaultdict import pprint from dataclasses import dataclass from enum import Enum import re from typing import Optional, Tuple import logkicker @dataclass class BlockReceived: cb_size_bytes: int = 0 cb_missing_bytes: int = 0 cb_missing_tx_count: int = 0 cb_bytes_from_extra_pool: Optional[int] = None @dataclass class BlockSent: block_received: BlockReceived peer_id: int cb_bytes_sent: int = 0 tcp_window_bytes: int = 0 class ReasonsToCare(Enum): WE_DONT = 0 CB_SEND = 1 CB_RECEIVE = 2 CB_RECONSTRUCTION = 3 CB_TO_ANNOUNCE = 4 CB_REQUESTED = 5 NET_MAX_SEND = 6 @dataclass(frozen=True) class LogPattern: """ Encapsulates a log pattern, its category, the reason it's important, and how to parse its captured data. """ reason: ReasonsToCare category: str regex: re.Pattern[str] # Maps a captured field name to a function for type conversion (e.g., int) # type_casts: Dict[str, TypeConverter] = field(default_factory=dict) LOG_PATTERNS = [ LogPattern( reason=ReasonsToCare.CB_RECONSTRUCTION, category="cmpctblock", # Successfully reconstructed block 000000000000000000000fec9bd60e4700c173a61195b46527bda8861f6b1276 with 1 txn prefilled, 4105 txn from mempool (incl at least 0 from extra pool) and 0 txn (0 bytes) requested regex=re.compile( r'Successfully reconstructed block (?P<blockhash>[0-9a-f]+) with ' r'(?P<prefill_count>\d+) txn prefilled, (?P<mempool_count>\d+) ' r'txn from mempool \(incl at least (?P<extrapool_count>\d+) from ' r'extra pool\) and (?P<requested_count>\d+) txn ' r'\((?P<requested_bytes>\d+) bytes\) requested' ), # type_casts={ # 'prefilled_txns': int, 'mempool_txns': int, 'extra_pool_txns': int, # 'requested_txns': int, 'requested_bytes': int # } ), LogPattern( reason=ReasonsToCare.CB_RECEIVE, category="cmpctblock", # Initialized PartiallyDownloadedBlock for block 00000000000000000002165564043bef508ec2a8ddf81e15916114cbb5ce632b using a cmpctblock of 14691 bytes regex=re.compile(r'Initialized PartiallyDownloadedBlock for block (?P<blockhash>[0-9a-f]+) using a cmpctblock of (?P<cmpctblock_bytes>\d+) bytes'), # type_casts={'cmpctblock_bytes': int} ), LogPattern( reason=ReasonsToCare.CB_SEND, category="net", # sending cmpctblock (25101 bytes) peer=1 regex=re.compile(r'sending cmpctblock \((?P<cmpctblock_bytes>\d+) bytes\) peer=(?P<peer_id>\d+)'), # type_casts={'cmpctblock_bytes': int, 'peer_id': int} ), LogPattern( reason=ReasonsToCare.CB_TO_ANNOUNCE, category="net", # PeerManager::NewPoWValidBlock sending header-and-ids 00000000000000000002165564043bef508ec2a8ddf81e15916114cbb5ce632b to peer=11 regex=re.compile(r'PeerManager::NewPoWValidBlock sending header-and-ids (?P<blockhash>[0-9a-f]+) to peer=(?P<peer_id>\d+)'), # type_casts={'peer_id': int} ), LogPattern( reason=ReasonsToCare.CB_REQUESTED, category="net", # received getdata for: cmpctblock 0000000000000000000085ae6fe4bb42bb2395c4fce575eac8f8dcaa8bea0750 peer=3 regex=re.compile(r'received getdata for: cmpctblock (?P<blockhash>[0-9a-f]+) peer=(?P<peer_id>\d+)'), # type_casts={'peer_id': int} ), LogPattern( reason=ReasonsToCare.NET_MAX_SEND, category="net", # - Max send per-rtt: 14480 bytes regex=re.compile(r'\s*- Max send per-rtt: (?P<max_send_bytes>\d+) bytes'), # We're gonna strip it # type_casts={'max_send_bytes': int} ), ] def we_care(entry: logkicker.LogEntry) -> Tuple[ReasonsToCare, Optional[dict]]: entry_category = entry.metadata.category for pattern in LOG_PATTERNS: if entry_category == pattern.category: if match := pattern.regex.match(entry.body): # Get a dictionary of {name: string_value} from the named # groups data = match.groupdict() # Apply type conversions for fields specified in the pattern # for field_name, cast_function in pattern.type_casts.items(): # if field_name in data: # data[field_name] = cast_function(data[field_name]) return pattern.reason, data return ReasonsToCare.WE_DONT, None def main(filepath): blocks_received: dict[str, BlockReceived] = {} blocks_sent: dict[str, list[BlockSent]] = defaultdict(list) pending_block_send: Optional[BlockSent] = None pending_max_send: Optional[BlockSent] = None for entry in logkicker.process_log_generator(filepath): why, match = we_care(entry) if why == ReasonsToCare.WE_DONT or match is None: continue match why: case ReasonsToCare.CB_RECEIVE: blocks_received[match['blockhash']] = BlockReceived(cb_size_bytes = int(match['cmpctblock_bytes'])) case ReasonsToCare.CB_RECONSTRUCTION: block = blocks_received[match['blockhash']] block.cb_missing_tx_count = int(match['requested_count']) block.cb_missing_bytes = int(match['requested_bytes']) case ReasonsToCare.CB_TO_ANNOUNCE | ReasonsToCare.CB_REQUESTED: # lucky, they have the same pattern! blockhash = match['blockhash'] # On rare occassions we receive full-sized blocks, so we don't know their cb size. if blockhash not in blocks_received: continue pending_block_send = BlockSent(block_received=blocks_received[blockhash], peer_id=int(match['peer_id'])) blocks_sent[blockhash].append(pending_block_send) case ReasonsToCare.CB_SEND: if not pending_block_send: continue assert pending_block_send.peer_id == int(match['peer_id']) pending_block_send.cb_bytes_sent = int(match['cmpctblock_bytes']) pending_max_send = pending_block_send pending_block_send = None case ReasonsToCare.NET_MAX_SEND: if not pending_max_send: continue pending_max_send.tcp_window_bytes = int(match['max_send_bytes']) pending_max_send = None # all of the below is data analysis, would be nice to separate parsing from analysis, maybe I should use pandas or one of these fancy python things? blocks_failing_reconstruction = [block for block in blocks_received.values() if block.cb_missing_tx_count > 0] print(f"{len(blocks_failing_reconstruction)} out of {len(blocks_received)} blocks received failed reconstruction.") fail_rate = len(blocks_failing_reconstruction) / len(blocks_received) reco_rate = 1 - fail_rate print(f"Reconstruction rate was {reco_rate * 100:.2f}%") total_cb_sent = total_prefilled_cb_sent = 0 total_prefill_bytes = total_prefill_extra_bytes = 0 total_available_bytes_in_all_windows = total_available_bytes_in_needed_windows = 0 # blocks where the whole prefill fit prefills_that_fit = 0 # blocks where the prefill without extra_txn fit no_extra_prefills_that_fit = 0 # blocks that exceeded the first rtt window without any prefills exceeded_without_prefill = 0 # Pretty print the blocks_sent data for blockhash, sent_list in blocks_sent.items(): if blockhash not in blocks_received: continue received = blocks_received[blockhash] # the size of the CMPCTBLOCK message we received. received_size = received.cb_size_bytes for sent in sent_list: # running total of CMPCTBLOCK messages sent total_cb_sent += 1 prefill_size = sent.cb_bytes_sent - received_size total_prefill_bytes += prefill_size # The number of rtt's it would have taken to announce the # CMPCTBLOCK with no additional prefilling rtts_needed_with_no_prefill: int = received_size // sent.tcp_window_bytes if (rtts_needed_with_no_prefill > 1): exceeded_without_prefill += 1 # Bytes used in this window before prefill bytes_used_in_tcp_window = received_size % sent.tcp_window_bytes # The number of bytes of overhead we have up to the next tcp window boundary bytes_left_in_tcp_window: int = sent.tcp_window_bytes - bytes_used_in_tcp_window # running total of tcp windows for stats total_available_bytes_in_all_windows += bytes_left_in_tcp_window if prefill_size > 0: # available total_available_bytes_in_needed_windows += bytes_left_in_tcp_window total_prefilled_cb_sent += 1 if prefill_size <= bytes_left_in_tcp_window: # we got it for free 😎 prefills_that_fit += 1 # this only works for the prefilling node. extra pool recovery size # is not logged, but we can recover this info from the difference # between sent size, todo: add logging to the branch # track the total number of cb's that needed prefills received.cb_bytes_from_extra_pool = sent.cb_bytes_sent - received_size - received.cb_missing_bytes total_prefill_extra_bytes += received.cb_bytes_from_extra_pool if prefill_size - received.cb_bytes_from_extra_pool <= bytes_left_in_tcp_window: # We probably win a reconstruction here, but more data is needed. no_extra_prefills_that_fit += 1 sent_per_block = total_cb_sent / len(blocks_received) print(f"{total_cb_sent} CMPCTBLOCK messages sent, {sent_per_block:.2f} per block.") already_over_rtt_rate = exceeded_without_prefill / total_cb_sent print(f"{exceeded_without_prefill}/{total_cb_sent} CMPCTBLOCK's sent were already over the window for a single RTT before prefilling. ({already_over_rtt_rate * 100:.2f}%)") avg_available_bytes = total_available_bytes_in_all_windows / total_cb_sent print(f"Avg available prefill bytes for all CMPCTBLOCK's we sent: {avg_available_bytes:.2f}") if total_prefill_bytes > 0: avg_available_bytes_for_needed = total_available_bytes_in_needed_windows / total_cb_sent print(f"Avg available prefill bytes for prefilled CMPCTBLOCK's we sent: {avg_available_bytes_for_needed:.2f}") avg_prefill_bytes = total_prefill_bytes / total_prefilled_cb_sent print(f"Avg total prefill size for CMPCTBLOCK's we prefilled: {avg_prefill_bytes:.2f}") avg_extra_prefill_bytes = total_prefill_extra_bytes / total_prefilled_cb_sent print(f"Avg bytes of prefill that were extra_txn: {avg_extra_prefill_bytes:.2f}") avg_prefill_without_extra = (total_prefill_bytes - total_prefill_extra_bytes) / total_prefilled_cb_sent print(f"Avg prefill size w/o extras: {avg_prefill_without_extra:.2f}") prefill_needed_rate = total_prefilled_cb_sent / total_cb_sent prefill_fit_rate = prefills_that_fit / total_prefilled_cb_sent print(f"{total_prefilled_cb_sent}/{total_cb_sent} blocks sent required prefills. ({prefill_needed_rate * 100:.2f}%)") print(f"{prefills_that_fit}/{total_prefilled_cb_sent} prefilled blocks sent fit in the available bytes. ({prefill_fit_rate * 100:.2f}%)") print(f"{no_extra_prefills_that_fit}/{total_prefilled_cb_sent} prefilled blocks would have fit if we hadn't prefilled extra txn's. ({no_extra_prefills_that_fit / total_prefilled_cb_sent * 100:.2f}%)") if __name__ == "__main__": import sys if len(sys.argv) != 2: print("Usage: python script.py <filepath>") sys.exit(1) filepath = sys.argv[1] entries = main(filepath) Assistant
That's a great question! You're right, the current script is excellent at parsing and doing some basic calculations, but it can definitely be enhanced for more sophisticated data analysis and presentation. The suggestion you had about
pandasand "fancy python things" is spot on.Here's how you can integrate
pandasfor analysis andmatplotlib/seabornfor pretty charts, along with a refactored structure for clarity.This approach involves:
- Collecting Data into Lists of Dictionaries: The parsing logic will be slightly adjusted to populate lists of simple dictionaries, which are ideal for converting into pandas DataFrames.
- Creating Pandas DataFrames: Convert the collected lists into structured DataFrames.
- Performing Analysis with Pandas: Use DataFrame operations for aggregation, filtering, and statistical calculations.
- Visualizing with Matplotlib/Seaborn: Generate various charts to illustrate the findings.
Let's break down the changes.
1. Required Libraries
First, ensure you have these libraries installed:
pip install pandas matplotlib seaborn logkicker
2. Refactoring the Code
We'll restructure the
mainfunction to: a. Parse logs and collect raw data into structured lists. b. Convert these lists into pandas DataFrames. c. Perform all analysis using DataFrame methods. d. Generate and display plots.#!/usr/bin/env python3 from collections import defaultdict import pprint from dataclasses import dataclass, field from enum import Enum import re from typing import Optional, Tuple, Dict, List, Any import logkicker import pandas as pd import matplotlib.pyplot as plt import seaborn as sns # --- Dataclasses and Enums (Keep as is) --- @dataclass class BlockReceived: cb_size_bytes: int = 0 cb_missing_bytes: int = 0 cb_missing_tx_count: int = 0 cb_bytes_from_extra_pool: Optional[int] = None # Add blockhash for easier DataFrame conversion blockhash: str = "" @dataclass class BlockSent: # Changed block_received to be a dictionary for easier DataFrame flattening block_received_data: Dict[str, Any] = field(default_factory=dict) peer_id: int = 0 cb_bytes_sent: int = 0 tcp_window_bytes: int = 0 # Add blockhash for easier DataFrame conversion blockhash: str = "" class ReasonsToCare(Enum): WE_DONT = 0 CB_SEND = 1 CB_RECEIVE = 2 CB_RECONSTRUCTION = 3 CB_TO_ANNOUNCE = 4 CB_REQUESTED = 5 NET_MAX_SEND = 6 @dataclass(frozen=True) class LogPattern: """ Encapsulates a log pattern, its category, the reason it's important, and how to parse its captured data. """ reason: ReasonsToCare category: str regex: re.Pattern[str] # type_casts: Dict[str, TypeConverter] = field(default_factory=dict) LOG_PATTERNS = [ LogPattern( reason=ReasonsToCare.CB_RECONSTRUCTION, category="cmpctblock", regex=re.compile( r'Successfully reconstructed block (?P<blockhash>[0-9a-f]+) with ' r'(?P<prefill_count>\d+) txn prefilled, (?P<mempool_count>\d+) ' r'txn from mempool \(incl at least (?P<extrapool_count>\d+) from ' r'extra pool\) and (?P<requested_count>\d+) txn ' r'\((?P<requested_bytes>\d+) bytes\) requested' ), ), LogPattern( reason=ReasonsToCare.CB_RECEIVE, category="cmpctblock", regex=re.compile(r'Initialized PartiallyDownloadedBlock for block (?P<blockhash>[0-9a-f]+) using a cmpctblock of (?P<cmpctblock_bytes>\d+) bytes'), ), LogPattern( reason=ReasonsToCare.CB_SEND, category="net", regex=re.compile(r'sending cmpctblock \((?P<cmpctblock_bytes>\d+) bytes\) peer=(?P<peer_id>\d+)'), ), LogPattern( reason=ReasonsToCare.CB_TO_ANNOUNCE, category="net", regex=re.compile(r'PeerManager::NewPoWValidBlock sending header-and-ids (?P<blockhash>[0-9a-f]+) to peer=(?P<peer_id>\d+)'), ), LogPattern( reason=ReasonsToCare.CB_REQUESTED, category="net", regex=re.compile(r'received getdata for: cmpctblock (?P<blockhash>[0-9a-f]+) peer=(?P<peer_id>\d+)'), ), LogPattern( reason=ReasonsToCare.NET_MAX_SEND, category="net", regex=re.compile(r'\s*- Max send per-rtt: (?P<max_send_bytes>\d+) bytes'), ), ] def we_care(entry: logkicker.LogEntry) -> Tuple[ReasonsToCare, Optional[Dict[str, str]]]: """ Checks if the log entry matches any of our patterns and returns the reason and extracted data. """ entry_category = entry.metadata.category for pattern in LOG_PATTERNS: if entry_category == pattern.category: if match := pattern.regex.match(entry.body): data = match.groupdict() # Apply type conversions directly here to simplify later processing # (This replaces the commented-out type_casts logic) for field_name, value in data.items(): try: data[field_name] = int(value) except ValueError: pass # Keep as string if not int return pattern.reason, data return ReasonsToCare.WE_DONT, None # --- Data Collection and DataFrame Creation --- def collect_parsed_data(filepath: str) -> Tuple[pd.DataFrame, pd.DataFrame]: """ Parses the log file and collects data into two pandas DataFrames: one for received blocks and one for sent blocks. """ received_data_list: List[Dict[str, Any]] = [] sent_data_list: List[Dict[str, Any]] = [] # Temporary storage for received block details, keyed by blockhash # { blockhash: { 'cb_size_bytes': ..., 'cb_missing_tx_count': ..., ... } } temp_received_blocks: Dict[str, Dict[str, Any]] = {} # State variables to track pending operations, mimicking original logic # `current_pending_block_send_obj`: The BlockSent object currently being built from ANNOUNCE/REQUEST -> SEND # `active_send_for_window_update`: Points to the specific BlockSent object waiting for a NET_MAX_SEND update. current_pending_block_send_obj: Optional[BlockSent] = None active_send_for_window_update: Optional[BlockSent] = None for entry in logkicker.process_log_generator(filepath): why, match = we_care(entry) if why == ReasonsToCare.WE_DONT or match is None: continue blockhash = match.get('blockhash') peer_id = match.get('peer_id', -1) # Use -1 if peer_id is not present if why == ReasonsToCare.CB_RECEIVE: bh = match['blockhash'] cb_size = match['cmpctblock_bytes'] # Store/update received block details if bh not in temp_received_blocks: temp_received_blocks[bh] = {'blockhash': bh} temp_received_blocks[bh]['cb_size_bytes'] = cb_size elif why == ReasonsToCare.CB_RECONSTRUCTION: bh = match['blockhash'] # Store/update reconstruction details if bh not in temp_received_blocks: temp_received_blocks[bh] = {'blockhash': bh} temp_received_blocks[bh]['cb_missing_tx_count'] = match.get('requested_count', 0) temp_received_blocks[bh]['cb_missing_bytes'] = match.get('requested_bytes', 0) temp_received_blocks[bh]['cb_bytes_from_extra_pool'] = match.get('extrapool_count', 0) elif why == ReasonsToCare.CB_TO_ANNOUNCE or why == ReasonsToCare.CB_REQUESTED: if blockhash and peer_id != -1: # Check if we have received block info for this blockhash if blockhash in temp_received_blocks: received_data = temp_received_blocks[blockhash] # Create a new BlockSent object for this event # Link it to the relevant received block data current_pending_block_send_obj = BlockSent( block_received_data=received_data, # Use the dict directly peer_id=peer_id, blockhash=blockhash, # Store blockhash in BlockSent too for easier DF creation cb_bytes_sent=0, # Default to 0, will be updated by CB_SEND tcp_window_bytes=0 # Default to 0, will be updated by NET_MAX_SEND ) sent_data_list.append(current_pending_block_send_obj) # This new BlockSent object is now the target for the next NET_MAX_SEND active_send_for_window_update = current_pending_block_send_obj # If blockhash is not in temp_received_blocks, we ignore this send event # as we don't have context for the received block. elif why == ReasonsToCare.CB_SEND: if peer_id != -1 and current_pending_block_send_obj is not None: # Check if the peer_id matches the current pending send operation if current_pending_block_send_obj.peer_id == peer_id: current_pending_block_send_obj.cb_bytes_sent = match['cmpctblock_bytes'] # The original code linked this send to the NET_MAX_SEND logic: # `pending_max_send = pending_block_send` # So, the *next* NET_MAX_SEND applies to *this* current_pending_block_send_obj active_send_for_window_update = current_pending_block_send_obj else: # This can happen if logs are out of order or a peer connection state is lost. # For simplicity, we might consider finding the most recent pending send for this peer. # However, sticking to the original code's apparent state management: # if current_pending_block_send_obj.peer_id != peer_id, we ignore this CB_SEND. pass # If current_pending_block_send_obj is None, or if peer_id doesn't match, we ignore. elif why == ReasonsToCare.NET_MAX_SEND: # This log line doesn't have a peer_id, implying it applies to the # currently active send operation tracked by `active_send_for_window_update`. if active_send_for_window_update is not None: active_send_for_window_update.tcp_window_bytes = match['max_send_bytes'] # Once applied, this window setting is consumed. active_send_for_window_update = None # If active_send_for_window_update is None, this NET_MAX_SEND is not relevant # to any tracked send operation. # --- Convert collected data into pandas DataFrames --- # 1. DataFrame for received blocks # Ensure all received blocks are added, even if they never got reconstruction details received_df = pd.DataFrame(list(temp_received_blocks.values())) # Fill missing columns with default values for consistency if they weren't in logs for col in ['cb_size_bytes', 'cb_missing_tx_count', 'cb_missing_bytes', 'cb_bytes_from_extra_pool']: if col not in received_df.columns: received_df[col] = 0 # Default to 0 for byte/count fields # Ensure blockhash is present if 'blockhash' not in received_df.columns: received_df['blockhash'] = None # Or handle as error if expected # 2. DataFrame for sent blocks # We need to flatten the nested `block_received_data` dictionary from BlockSent objects flattened_sent_data = [] for bs in sent_data_list: if not bs.block_received_data: # Skip if no received data was linked (shouldn't happen if logic is correct) continue record = { 'sent_peer_id': bs.peer_id, 'sent_cb_bytes_sent': bs.cb_bytes_sent, 'sent_tcp_window_bytes': bs.tcp_window_bytes, 'sent_blockhash': bs.blockhash, # Already added in BlockSent } # Add fields from the nested received data dictionary for key, value in bs.block_received_data.items(): record[f'received_{key}'] = value flattened_sent_data.append(record) sent_df = pd.DataFrame(flattened_sent_data) # Fill missing columns in sent_df as well for col in ['sent_cb_bytes_sent', 'sent_tcp_window_bytes', 'sent_peer_id']: if col not in sent_df.columns: sent_df[col] = 0 for col in ['received_cb_size_bytes', 'received_cb_missing_tx_count', 'received_cb_missing_bytes', 'received_cb_bytes_from_extra_pool']: if col not in sent_df.columns: sent_df[col] = 0 # Ensure blockhash is in both DFs for potential merging, though not strictly needed for current analysis if 'blockhash' not in sent_df.columns and 'sent_blockhash' in sent_df.columns: sent_df['blockhash'] = sent_df['sent_blockhash'] # Alias for consistency if needed return received_df, sent_df # --- Data Analysis with Pandas --- def analyze_cmpctblock_data(received_df: pd.DataFrame, sent_df: pd.DataFrame) -> Dict[str, Any]: """ Performs statistical analysis on the received and sent data using pandas. Returns a dictionary of key metrics. """ analysis_results = {} # --- Analysis for Received Blocks --- total_received = len(received_df) if total_received > 0: received_df['failed_reconstruction'] = received_df['cb_missing_tx_count'] > 0 failed_reconstruction_count = received_df['failed_reconstruction'].sum() analysis_results['total_blocks_received'] = total_received analysis_results['blocks_failed_reconstruction'] = failed_reconstruction_count analysis_results['reconstruction_rate'] = 1.0 - (failed_reconstruction_count / total_received) else: analysis_results['total_blocks_received'] = 0 analysis_results['blocks_failed_reconstruction'] = 0 analysis_results['reconstruction_rate'] = 0.0 # --- Analysis for Sent Blocks --- if not sent_df.empty: # Calculate prefill bytes sent_df['prefill_bytes'] = sent_df['sent_cb_bytes_sent'] - sent_df['received_cb_size_bytes'] # Identify blocks that needed prefilling sent_df['requires_prefill'] = sent_df['prefill_bytes'] > 0 prefilled_count = sent_df['requires_prefill'].sum() analysis_results['total_cmpctblock_messages_sent'] = len(sent_df) analysis_results['avg_sent_cmpctblock_bytes'] = sent_df['sent_cb_bytes_sent'].mean() analysis_results['blocks_requiring_prefill_count'] = prefilled_count if prefilled_count > 0: analysis_results['avg_prefill_bytes'] = sent_df[sent_df['requires_prefill']]['prefill_bytes'].mean() analysis_results['avg_extra_prefill_bytes'] = sent_df[sent_df['requires_prefill']]['received_cb_bytes_from_extra_pool'].mean() # Calculate prefill size without extras sent_df['prefill_base_bytes'] = sent_df['prefill_bytes'] - sent_df.get('received_cb_bytes_from_extra_pool', 0) sent_df['prefill_base_bytes'] = sent_df['prefill_base_bytes'].clip(lower=0) # Ensure non-negative analysis_results['avg_prefill_bytes_without_extra'] = sent_df[sent_df['requires_prefill']]['prefill_base_bytes'].mean() # Calculate window availability and fit sent_df['bytes_used_in_window'] = sent_df['received_cb_size_bytes'] % sent_df['sent_tcp_window_bytes'] sent_df['bytes_left_in_window'] = sent_df['sent_tcp_window_bytes'] - sent_df['bytes_used_in_window'] # Correct calculation for bytes_left_in_window when received_cb_size_bytes is a multiple of tcp_window_bytes # If bytes_used_in_window is 0, it means it perfectly filled a window, so the *next* window is fully available. # Or, more accurately, if it *just* filled a window, there are no bytes left in *that* window. # The original code's logic `sent.tcp_window_bytes - bytes_used_in_tcp_window` seems correct. # If bytes_used_in_window is 0, then bytes_left_in_window = tcp_window_bytes. analysis_results['avg_available_prefill_bytes_all_sends'] = sent_df['bytes_left_in_window'].mean() analysis_results['avg_available_prefill_bytes_for_prefilled'] = sent_df[sent_df['requires_prefill']]['bytes_left_in_window'].mean() # Prefill fit calculations sent_df['prefill_fits_in_window'] = sent_df['prefill_bytes'] <= sent_df['bytes_left_in_window'] analysis_results['prefill_fit_rate'] = sent_df[sent_df['requires_prefill']]['prefill_fits_in_window'].mean() # Proportion of prefilled blocks that fit sent_df['prefill_fits_if_no_extra'] = sent_df['prefill_base_bytes'] <= sent_df['bytes_left_in_window'] analysis_results['prefill_fit_if_no_extra_rate'] = sent_df[sent_df['requires_prefill']]['prefill_fits_if_no_extra'].mean() # Calculate blocks exceeding RTT without prefill # Original logic: `rtts_needed_with_no_prefill: int = received_size // sent.tcp_window_bytes` sent_df['rtts_needed_base'] = sent_df['received_cb_size_bytes'] // sent_df['sent_tcp_window_bytes'] exceeded_without_prefill_count = (sent_df['rtts_needed_base'] > 1).sum() # More than 1 RTT needed for base block analysis_results['exceeded_single_rtt_count'] = exceeded_without_prefill_count analysis_results['exceeded_single_rtt_rate'] = exceeded_without_prefill_count / len(sent_df) if len(sent_df) else 0 else: analysis_results['avg_prefill_bytes'] = 0 analysis_results['avg_extra_prefill_bytes'] = 0 analysis_results['avg_prefill_bytes_without_extra'] = 0 analysis_results['avg_available_prefill_bytes_all_sends'] = 0 analysis_results['avg_available_prefill_bytes_for_prefilled'] = 0 analysis_results['prefill_fit_rate'] = 0 analysis_results['prefill_fit_if_no_extra_rate'] = 0 analysis_results['exceeded_single_rtt_count'] = 0 analysis_results['exceeded_single_rtt_rate'] = 0 else: # No sent data found analysis_results['total_cmpctblock_messages_sent'] = 0 analysis_results['avg_sent_cmpctblock_bytes'] = 0 analysis_results['blocks_requiring_prefill_count'] = 0 # Set all other sent data metrics to 0 or None for key in ['avg_prefill_bytes', 'avg_extra_prefill_bytes', 'avg_prefill_bytes_without_extra', 'avg_available_prefill_bytes_all_sends', 'avg_available_prefill_bytes_for_prefilled', 'prefill_fit_rate', 'prefill_fit_if_no_extra_rate', 'exceeded_single_rtt_count', 'exceeded_single_rtt_rate']: analysis_results[key] = 0 return analysis_results # --- Visualization --- def plot_results(received_df: pd.DataFrame, sent_df: pd.DataFrame, analysis_results: Dict[str, Any]): """ Generates and displays plots based on the analysis results and DataFrames. """ sns.set_theme(style="whitegrid") # Plot 1: Block Reconstruction Success Rate if analysis_results.get('total_blocks_received', 0) > 0: plt.figure(figsize=(7, 5)) labels = ['Successful', 'Failed'] counts = [ analysis_results['total_blocks_received'] - analysis_results['blocks_failed_reconstruction'], analysis_results['blocks_failed_reconstruction'] ] sns.barplot(x=labels, y=counts, palette="viridis") plt.title('Block Reconstruction Success Rate') plt.ylabel('Number of Blocks') plt.xlabel('Reconstruction Status') # Add text labels on bars for i, count in enumerate(counts): plt.text(i, count + 0.5, str(count), ha='center', va='bottom') plt.tight_layout() plt.show() else: print("No received block data to plot reconstruction rate.") # Plot 2: Distribution of received CMPCTBLOCK sizes if not received_df.empty and 'cb_size_bytes' in received_df.columns: plt.figure(figsize=(10, 6)) sns.histplot(received_df['cb_size_bytes'], bins=50, kde=True, color="skyblue") plt.title('Distribution of Received CMPCTBLOCK Sizes') plt.xlabel('Bytes') plt.ylabel('Frequency') plt.tight_layout() plt.show() else: print("No received CMPCTBLOCK sizes to plot.") # Plot 3: Distribution of sent CMPCTBLOCK sizes (when prefilled) if not sent_df.empty and analysis_results.get('blocks_requiring_prefill_count', 0) > 0: plt.figure(figsize=(10, 6)) sns.histplot(sent_df[sent_df['requires_prefill']]['sent_cb_bytes_sent'], bins=50, kde=True, color="lightcoral") plt.title('Distribution of Sent CMPCTBLOCK Sizes (Prefilled)') plt.xlabel('Bytes') plt.ylabel('Frequency') plt.tight_layout() plt.show() else: print("No prefilled sent CMPCTBLOCK sizes to plot.") # Plot 4: Prefill Bytes vs. Available Bytes in Window if not sent_df.empty and analysis_results.get('blocks_requiring_prefill_count', 0) > 0: plot_df = sent_df[sent_df['requires_prefill']].copy() # Ensure 'bytes_left_in_window' is calculated even if it was missing if 'bytes_left_in_window' not in plot_df.columns: plot_df['bytes_left_in_window'] = plot_df['sent_tcp_window_bytes'] - (plot_df['received_cb_size_bytes'] % plot_df['sent_tcp_window_bytes']) plot_df['bytes_left_in_window'] = plot_df['bytes_left_in_window'].clip(lower=0) # Ensure non-negative plt.figure(figsize=(10, 6)) sns.scatterplot(data=plot_df, x='bytes_left_in_window', y='prefill_bytes', hue='prefill_fits_in_window', s=60, palette="viridis", alpha=0.7) plt.title('Prefill Bytes vs. Available Window Bytes') plt.xlabel('Available Bytes in TCP Window') plt.ylabel('Prefill Bytes Used') plt.legend(title='Fits in Window', loc='upper left') plt.tight_layout() plt.show() else: print("No data for Prefill Bytes vs. Available Window Bytes plot.") # Plot 5: Prefill Success Rate vs. RTTs Needed for Base Block if not sent_df.empty and analysis_results.get('blocks_requiring_prefill_count', 0) > 0: plot_df_rtt = sent_df[sent_df['requires_prefill']].copy() if 'rtts_needed_base' not in plot_df_rtt.columns: plot_df_rtt['rtts_needed_base'] = plot_df_rtt['received_cb_size_bytes'] // plot_df_rtt['sent_tcp_window_bytes'] plot_df_rtt['rtts_needed_base'] = plot_df_rtt['rtts_needed_base'].clip(lower=0) # Map boolean to string for legend plot_df_rtt['fits_in_window_label'] = plot_df_rtt['prefill_fits_in_window'].map({True: 'Fits', False: 'Exceeds'}) plt.figure(figsize=(10, 6)) sns.scatterplot(data=plot_df_rtt, x='rtts_needed_base', y='prefill_bytes', hue='fits_in_window_label', s=60, palette="viridis", alpha=0.7) plt.title('Prefill Bytes vs. RTTs Needed (for base CMPCTBLOCK)') plt.xlabel('RTTs needed for base CMPCTBLOCK') plt.ylabel('Prefill Bytes Used') plt.legend(title='Prefill Status', loc='upper right') plt.tight_layout() plt.show() else: print("No data for Prefill Success Rate vs. RTTs Needed plot.") # Plot 6: Distribution of Prefill Bytes (Base vs. Extra Pool) if not sent_df.empty and analysis_results.get('blocks_requiring_prefill_count', 0) > 0: plot_df_extra = sent_df[sent_df['requires_prefill']].copy() # Ensure 'prefill_base_bytes' is calculated if 'prefill_base_bytes' not in plot_df_extra.columns: plot_df_extra['prefill_base_bytes'] = plot_df_extra['prefill_bytes'] - plot_df_extra.get('received_cb_bytes_from_extra_pool', 0) plot_df_extra['prefill_base_bytes'] = plot_df_extra['prefill_base_bytes'].clip(lower=0) # Melt for stacked bar/histplot melted_df_extra = plot_df_extra.melt( value_vars=['prefill_base_bytes', 'received_cb_bytes_from_extra_pool'], var_name='Prefill Type', value_name='Bytes' ) # Rename categories for better legend labels melted_df_extra['Prefill Type'] = melted_df_extra['Prefill Type'].replace({ 'prefill_base_bytes': 'Base Prefill', 'received_cb_bytes_from_extra_pool': 'Extra Pool Prefill' }) plt.figure(figsize=(10, 6)) # Using histplot with 'multiple="stack"' to show the breakdown sns.histplot(data=melted_df_extra, x='Bytes', hue='Prefill Type', weights='Bytes', multiple="stack", bins=50, # Use weights for total bytes distribution palette="tab10", alpha=0.8) plt.title('Distribution of Prefill Bytes (Base vs. Extra Pool)') plt.xlabel('Bytes') plt.ylabel('Total Bytes in Bin') # Clarify Y-axis meaning for stacked histplot plt.tight_layout() plt.show() else: print("No data for Prefill Bytes Distribution plot.") # Plot 7: CMPCTBLOCK messages sent vs. base CMPCTBLOCK size if not sent_df.empty: plt.figure(figsize=(10, 6)) sns.scatterplot(data=sent_df, x='received_cb_size_bytes', y='sent_cb_bytes_sent', hue='requires_prefill', s=60, palette="viridis", alpha=0.7) plt.title('Sent CMPCTBLOCK Size vs. Received CMPCTBLOCK Size') plt.xlabel('Received CMPCTBLOCK Size (Bytes)') plt.ylabel('Sent CMPCTBLOCK Size (Bytes)') plt.legend(title='Prefill Required', loc='upper left') plt.tight_layout() plt.show() # --- Main Execution Flow --- def main(filepath): """ Orchestrates the log parsing, data analysis, and visualization. """ print(f"Processing log file: {filepath}") try: received_df, sent_df = collect_parsed_data(filepath) except Exception as e: print(f"Error during data collection: {e}") return print(f"\n--- Analysis Summary ---") analysis_results = analyze_cmpctblock_data(received_df, sent_df) # Print summary statistics print(f"Total blocks received: {analysis_results.get('total_blocks_received', 0)}") print(f"Blocks failed reconstruction: {analysis_results.get('blocks_failed_reconstruction', 0)}") print(f"Reconstruction rate: {analysis_results.get('reconstruction_rate', 0.0) * 100:.2f}%") print("-" * 20) total_sent = analysis_results.get('total_cmpctblock_messages_sent', 0) if total_sent > 0: print(f"Total CMPCTBLOCK messages sent: {total_sent}") print(f"Avg CMPCTBLOCK bytes sent per message: {analysis_results.get('avg_sent_cmpctblock_bytes', 0):.2f}") print(f"Blocks requiring prefill: {analysis_results.get('blocks_requiring_prefill_count', 0)} ({analysis_results.get('blocks_requiring_prefill_count', 0)/total_sent:.1%})") print(f"Avg prefill bytes (when prefilled): {analysis_results.get('avg_prefill_bytes', 0):.2f}") print(f"Avg extra pool prefill bytes: {analysis_results.get('avg_extra_prefill_bytes', 0):.2f}") print(f"Avg base prefill bytes (w/o extra): {analysis_results.get('avg_prefill_bytes_without_extra', 0):.2f}") print(f"Prefill fits in window rate: {analysis_results.get('prefill_fit_rate', 0.0) * 100:.2f}%") print(f"Prefill fits if no extra rate: {analysis_results.get('prefill_fit_if_no_extra_rate', 0.0) * 100:.2f}%") print(f"Blocks exceeding single RTT (base): {analysis_results.get('exceeded_single_rtt_count', 0)} ({analysis_results.get('exceeded_single_rtt_rate', 0.0) * 100:.2f}%)") print(f"Avg available prefill bytes (all sends): {analysis_results.get('avg_available_prefill_bytes_all_sends', 0):.2f}") print(f"Avg available prefill bytes (for prefilled): {analysis_results.get('avg_available_prefill_bytes_for_prefilled', 0):.2f}") else: print("No CMPCTBLOCK send data found.") print(f"\n--- Generating Plots ---") plot_results(received_df, sent_df, analysis_results) if __name__ == "__main__": import sys if len(sys.argv) != 2: print("Usage: python your_script_name.py <filepath>") sys.exit(1) filepath = sys.argv[1] main(filepath)
Explanation of Changes:
-
collect_parsed_data(filepath)Function:- This function replaces the parsing logic from
main. - It initializes
temp_received_blocks(a dictionary to hold details about received blocks, keyed byblockhash) andsent_data_list(a list to holdBlockSentobjects). - It uses the same
we_carelogic to identify relevant log lines. - State Management: The logic for
current_pending_block_send_objandactive_send_for_window_updatemirrors the original script's attempt to link sequential log entries (CB_TO_ANNOUNCE/REQUESTED->CB_SEND->NET_MAX_SEND). This is crucial for correctly populating thesent_data_list. - Data Structures:
BlockReceiveddataclass is modified to includeblockhash.BlockSentis modified to storeblock_received_dataas a dictionary and alsoblockhashdirectly. This makes flattening for DataFrames easier. - Output: Returns two pandas DataFrames:
received_dfandsent_df. - Type Conversion: Moved type conversions (
int()) directly intowe_carefor simplicity when creatingmatchdictionaries.
- This function replaces the parsing logic from
-
analyze_cmpctblock_data(received_df, sent_df)Function:- This function takes the two DataFrames as input.
- It leverages pandas' powerful methods:
- Boolean indexing (
sent_df[sent_df['requires_prefill']]) to filter rows. .sum(),.mean()for calculations.- Creating new columns (
df['new_col'] = ...) for derived metrics likeprefill_bytes,failed_reconstruction,bytes_left_in_window, etc. .clip(lower=0)is used to ensure calculated byte counts aren't negative..map()to convert boolean flags to readable strings for plots.
- Boolean indexing (
- Output: Returns a dictionary (
analysis_results) containing all the calculated summary statistics.
-
plot_results(received_df, sent_df, analysis_results)Function:- Takes the DataFrames and the analysis results dictionary.
- Uses
matplotlib.pyplotandseabornto generate plots:- Reconstruction Success Rate: A simple bar chart showing the counts of successful vs. failed reconstructions.
- Received CMPCTBLOCK Sizes: A histogram showing the distribution of sizes of the compressed blocks received.
- Sent CMPCTBLOCK Sizes (Prefilled): A histogram showing the distribution of sizes for blocks that required prefilling.
- Prefill Bytes vs. Available Window Bytes: A scatter plot showing how many prefill bytes were used versus how many bytes were available in the TCP window for that transmission. Points are colored based on whether the prefill fit.
- Prefill Bytes vs. RTTs Needed: Another scatter plot showing prefill bytes against the theoretical RTTs needed for the base block, highlighting when prefilling became necessary.
- Prefill Bytes Distribution (Base vs. Extra Pool): A stacked histogram showing the breakdown of prefill bytes into those from the base block (if it contained prefilled txns) and those specifically from the extra pool.
- Sent vs. Received Size: A scatter plot showing the relationship between the size of the compressed block received and the size of the cmpctblock message sent.
sns.set_theme(style="whitegrid")provides a nice default aesthetic.plt.tight_layout()adjusts plot parameters for a tight layout.plt.show()displays each plot.
-
main(filepath)Function:- This is now an orchestrator.
- It calls
collect_parsed_datato get the DataFrames. - It calls
analyze_cmpctblock_datato get the metrics and prints a summary of these metrics. - It calls
plot_resultsto generate the visualizations. - Includes basic error handling for the data collection phase.
How to Use:
- Save the modified code as a Python file (e.g.,
analyze_logs.py). - Run it from your terminal, passing the path to your log file as an argument:
python analyze_logs.py /path/to/your/bitcoin_node.log
This setup provides a much more robust and visually insightful way to analyze your log data, allowing you to quickly grasp the behavior of the compact block protocol.
Other models, same or similar prompt
Responses were produced in different, uncontrolled contexts (system prompts, settings, prior turns). This is not a benchmark.