Is there a nice way for me to be doing the data analysis on this and making pretty…
- User
Is there a nice way for me to be doing the data analysis on this and making pretty charts,m etc.:
#!/usr/bin/env python3 from collections import defaultdict import pprint from dataclasses import dataclass from enum import Enum import re from typing import Optional, Tuple import logkicker @dataclass class BlockReceived: cb_size_bytes: int = 0 cb_missing_bytes: int = 0 cb_missing_tx_count: int = 0 cb_bytes_from_extra_pool: Optional[int] = None @dataclass class BlockSent: block_received: BlockReceived peer_id: int cb_bytes_sent: int = 0 tcp_window_bytes: int = 0 class ReasonsToCare(Enum): WE_DONT = 0 CB_SEND = 1 CB_RECEIVE = 2 CB_RECONSTRUCTION = 3 CB_TO_ANNOUNCE = 4 CB_REQUESTED = 5 NET_MAX_SEND = 6 @dataclass(frozen=True) class LogPattern: """ Encapsulates a log pattern, its category, the reason it's important, and how to parse its captured data. """ reason: ReasonsToCare category: str regex: re.Pattern[str] # Maps a captured field name to a function for type conversion (e.g., int) # type_casts: Dict[str, TypeConverter] = field(default_factory=dict) LOG_PATTERNS = [ LogPattern( reason=ReasonsToCare.CB_RECONSTRUCTION, category="cmpctblock", # Successfully reconstructed block 000000000000000000000fec9bd60e4700c173a61195b46527bda8861f6b1276 with 1 txn prefilled, 4105 txn from mempool (incl at least 0 from extra pool) and 0 txn (0 bytes) requested regex=re.compile( r'Successfully reconstructed block (?P<blockhash>[0-9a-f]+) with ' r'(?P<prefill_count>\d+) txn prefilled, (?P<mempool_count>\d+) ' r'txn from mempool \(incl at least (?P<extrapool_count>\d+) from ' r'extra pool\) and (?P<requested_count>\d+) txn ' r'\((?P<requested_bytes>\d+) bytes\) requested' ), # type_casts={ # 'prefilled_txns': int, 'mempool_txns': int, 'extra_pool_txns': int, # 'requested_txns': int, 'requested_bytes': int # } ), LogPattern( reason=ReasonsToCare.CB_RECEIVE, category="cmpctblock", # Initialized PartiallyDownloadedBlock for block 00000000000000000002165564043bef508ec2a8ddf81e15916114cbb5ce632b using a cmpctblock of 14691 bytes regex=re.compile(r'Initialized PartiallyDownloadedBlock for block (?P<blockhash>[0-9a-f]+) using a cmpctblock of (?P<cmpctblock_bytes>\d+) bytes'), # type_casts={'cmpctblock_bytes': int} ), LogPattern( reason=ReasonsToCare.CB_SEND, category="net", # sending cmpctblock (25101 bytes) peer=1 regex=re.compile(r'sending cmpctblock \((?P<cmpctblock_bytes>\d+) bytes\) peer=(?P<peer_id>\d+)'), # type_casts={'cmpctblock_bytes': int, 'peer_id': int} ), LogPattern( reason=ReasonsToCare.CB_TO_ANNOUNCE, category="net", # PeerManager::NewPoWValidBlock sending header-and-ids 00000000000000000002165564043bef508ec2a8ddf81e15916114cbb5ce632b to peer=11 regex=re.compile(r'PeerManager::NewPoWValidBlock sending header-and-ids (?P<blockhash>[0-9a-f]+) to peer=(?P<peer_id>\d+)'), # type_casts={'peer_id': int} ), LogPattern( reason=ReasonsToCare.CB_REQUESTED, category="net", # received getdata for: cmpctblock 0000000000000000000085ae6fe4bb42bb2395c4fce575eac8f8dcaa8bea0750 peer=3 regex=re.compile(r'received getdata for: cmpctblock (?P<blockhash>[0-9a-f]+) peer=(?P<peer_id>\d+)'), # type_casts={'peer_id': int} ), LogPattern( reason=ReasonsToCare.NET_MAX_SEND, category="net", # - Max send per-rtt: 14480 bytes regex=re.compile(r'\s*- Max send per-rtt: (?P<max_send_bytes>\d+) bytes'), # We're gonna strip it # type_casts={'max_send_bytes': int} ), ] def we_care(entry: logkicker.LogEntry) -> Tuple[ReasonsToCare, Optional[dict]]: entry_category = entry.metadata.category for pattern in LOG_PATTERNS: if entry_category == pattern.category: if match := pattern.regex.match(entry.body): # Get a dictionary of {name: string_value} from the named # groups data = match.groupdict() # Apply type conversions for fields specified in the pattern # for field_name, cast_function in pattern.type_casts.items(): # if field_name in data: # data[field_name] = cast_function(data[field_name]) return pattern.reason, data return ReasonsToCare.WE_DONT, None def main(filepath): blocks_received: dict[str, BlockReceived] = {} blocks_sent: dict[str, list[BlockSent]] = defaultdict(list) pending_block_send: Optional[BlockSent] = None pending_max_send: Optional[BlockSent] = None for entry in logkicker.process_log_generator(filepath): why, match = we_care(entry) if why == ReasonsToCare.WE_DONT or match is None: continue match why: case ReasonsToCare.CB_RECEIVE: blocks_received[match['blockhash']] = BlockReceived(cb_size_bytes = int(match['cmpctblock_bytes'])) case ReasonsToCare.CB_RECONSTRUCTION: block = blocks_received[match['blockhash']] block.cb_missing_tx_count = int(match['requested_count']) block.cb_missing_bytes = int(match['requested_bytes']) case ReasonsToCare.CB_TO_ANNOUNCE | ReasonsToCare.CB_REQUESTED: # lucky, they have the same pattern! blockhash = match['blockhash'] # On rare occassions we receive full-sized blocks, so we don't know their cb size. if blockhash not in blocks_received: continue pending_block_send = BlockSent(block_received=blocks_received[blockhash], peer_id=int(match['peer_id'])) blocks_sent[blockhash].append(pending_block_send) case ReasonsToCare.CB_SEND: if not pending_block_send: continue assert pending_block_send.peer_id == int(match['peer_id']) pending_block_send.cb_bytes_sent = int(match['cmpctblock_bytes']) pending_max_send = pending_block_send pending_block_send = None case ReasonsToCare.NET_MAX_SEND: if not pending_max_send: continue pending_max_send.tcp_window_bytes = int(match['max_send_bytes']) pending_max_send = None # all of the below is data analysis, would be nice to separate parsing from analysis, maybe I should use pandas or one of these fancy python things? blocks_failing_reconstruction = [block for block in blocks_received.values() if block.cb_missing_tx_count > 0] print(f"{len(blocks_failing_reconstruction)} out of {len(blocks_received)} blocks received failed reconstruction.") fail_rate = len(blocks_failing_reconstruction) / len(blocks_received) reco_rate = 1 - fail_rate print(f"Reconstruction rate was {reco_rate * 100:.2f}%") total_cb_sent = total_prefilled_cb_sent = 0 total_prefill_bytes = total_prefill_extra_bytes = 0 total_available_bytes_in_all_windows = total_available_bytes_in_needed_windows = 0 # blocks where the whole prefill fit prefills_that_fit = 0 # blocks where the prefill without extra_txn fit no_extra_prefills_that_fit = 0 # blocks that exceeded the first rtt window without any prefills exceeded_without_prefill = 0 # Pretty print the blocks_sent data for blockhash, sent_list in blocks_sent.items(): if blockhash not in blocks_received: continue received = blocks_received[blockhash] # the size of the CMPCTBLOCK message we received. received_size = received.cb_size_bytes for sent in sent_list: # running total of CMPCTBLOCK messages sent total_cb_sent += 1 prefill_size = sent.cb_bytes_sent - received_size total_prefill_bytes += prefill_size # The number of rtt's it would have taken to announce the # CMPCTBLOCK with no additional prefilling rtts_needed_with_no_prefill: int = received_size // sent.tcp_window_bytes if (rtts_needed_with_no_prefill > 1): exceeded_without_prefill += 1 # Bytes used in this window before prefill bytes_used_in_tcp_window = received_size % sent.tcp_window_bytes # The number of bytes of overhead we have up to the next tcp window boundary bytes_left_in_tcp_window: int = sent.tcp_window_bytes - bytes_used_in_tcp_window # running total of tcp windows for stats total_available_bytes_in_all_windows += bytes_left_in_tcp_window if prefill_size > 0: # available total_available_bytes_in_needed_windows += bytes_left_in_tcp_window total_prefilled_cb_sent += 1 if prefill_size <= bytes_left_in_tcp_window: # we got it for free 😎 prefills_that_fit += 1 # this only works for the prefilling node. extra pool recovery size # is not logged, but we can recover this info from the difference # between sent size, todo: add logging to the branch # track the total number of cb's that needed prefills received.cb_bytes_from_extra_pool = sent.cb_bytes_sent - received_size - received.cb_missing_bytes total_prefill_extra_bytes += received.cb_bytes_from_extra_pool if prefill_size - received.cb_bytes_from_extra_pool <= bytes_left_in_tcp_window: # We probably win a reconstruction here, but more data is needed. no_extra_prefills_that_fit += 1 sent_per_block = total_cb_sent / len(blocks_received) print(f"{total_cb_sent} CMPCTBLOCK messages sent, {sent_per_block:.2f} per block.") already_over_rtt_rate = exceeded_without_prefill / total_cb_sent print(f"{exceeded_without_prefill}/{total_cb_sent} CMPCTBLOCK's sent were already over the window for a single RTT before prefilling. ({already_over_rtt_rate * 100:.2f}%)") avg_available_bytes = total_available_bytes_in_all_windows / total_cb_sent print(f"Avg available prefill bytes for all CMPCTBLOCK's we sent: {avg_available_bytes:.2f}") if total_prefill_bytes > 0: avg_available_bytes_for_needed = total_available_bytes_in_needed_windows / total_cb_sent print(f"Avg available prefill bytes for prefilled CMPCTBLOCK's we sent: {avg_available_bytes_for_needed:.2f}") avg_prefill_bytes = total_prefill_bytes / total_prefilled_cb_sent print(f"Avg total prefill size for CMPCTBLOCK's we prefilled: {avg_prefill_bytes:.2f}") avg_extra_prefill_bytes = total_prefill_extra_bytes / total_prefilled_cb_sent print(f"Avg bytes of prefill that were extra_txn: {avg_extra_prefill_bytes:.2f}") avg_prefill_without_extra = (total_prefill_bytes - total_prefill_extra_bytes) / total_prefilled_cb_sent print(f"Avg prefill size w/o extras: {avg_prefill_without_extra:.2f}") prefill_needed_rate = total_prefilled_cb_sent / total_cb_sent prefill_fit_rate = prefills_that_fit / total_prefilled_cb_sent print(f"{total_prefilled_cb_sent}/{total_cb_sent} blocks sent required prefills. ({prefill_needed_rate * 100:.2f}%)") print(f"{prefills_that_fit}/{total_prefilled_cb_sent} prefilled blocks sent fit in the available bytes. ({prefill_fit_rate * 100:.2f}%)") print(f"{no_extra_prefills_that_fit}/{total_prefilled_cb_sent} prefilled blocks would have fit if we hadn't prefilled extra txn's. ({no_extra_prefills_that_fit / total_prefilled_cb_sent * 100:.2f}%)") if __name__ == "__main__": import sys if len(sys.argv) != 2: print("Usage: python script.py <filepath>") sys.exit(1) filepath = sys.argv[1] entries = main(filepath) Assistant
Improving Data Analysis and Visualization
Your code is well-structured for parsing Bitcoin compact block logs, but you're right that the analysis and visualization could be improved. Here's how you can enhance it using pandas, matplotlib, and seaborn for better data analysis and visualization.
Recommended Improvements
1. Separate Parsing from Analysis
First, let's restructure your code to separate the parsing logic from the analysis:
def parse_logs(filepath): """Parse the log file and return structured data.""" blocks_received = {} blocks_sent = defaultdict(list) pending_block_send = None pending_max_send = None for entry in logkicker.process_log_generator(filepath): why, match = we_care(entry) if why == ReasonsToCare.WE_DONT or match is None: continue # [Rest of your parsing logic...] return { 'blocks_received': blocks_received, 'blocks_sent': blocks_sent }2. Use Pandas for Data Analysis
Convert your data structures to pandas DataFrames for easier analysis:
import pandas as pd def analyze_data(parsed_data): """Convert parsed data to DataFrames and perform analysis.""" # Convert blocks_received to DataFrame blocks_received = parsed_data['blocks_received'] received_df = pd.DataFrame.from_dict(blocks_received, orient='index') received_df.index.name = 'blockhash' received_df.reset_index(inplace=True) # Convert blocks_sent to DataFrame sent_data = [] for blockhash, sent_list in parsed_data['blocks_sent'].items(): for sent in sent_list: sent_data.append({ 'blockhash': blockhash, 'peer_id': sent.peer_id, 'cb_bytes_sent': sent.cb_bytes_sent, 'tcp_window_bytes': sent.tcp_window_bytes, 'cb_size_bytes': sent.block_received.cb_size_bytes, 'cb_missing_bytes': sent.block_received.cb_missing_bytes, 'cb_missing_tx_count': sent.block_received.cb_missing_tx_count, 'cb_bytes_from_extra_pool': sent.block_received.cb_bytes_from_extra_pool }) sent_df = pd.DataFrame(sent_data) return { 'received_df': received_df, 'sent_df': sent_df }3. Create Visualizations
Now let's create some meaningful visualizations:
import matplotlib.pyplot as plt import seaborn as sns import numpy as np def create_visualizations(dataframes): """Create visualizations from the analyzed data.""" received_df = dataframes['received_df'] sent_df = dataframes['sent_df'] # Set style for better looking plots sns.set_style("whitegrid") plt.figure(figsize=(12, 8)) # 1. Reconstruction success/failure plt.subplot(2, 2, 1) reco_counts = received_df['cb_missing_tx_count'].apply(lambda x: x > 0).value_counts() plt.pie(reco_counts, labels=['Successful', 'Failed'], autopct='%1.1f%%', colors=['#4CAF50', '#F44336']) plt.title('Block Reconstruction Success Rate') # 2. Distribution of compact block sizes plt.subplot(2, 2, 2) sns.histplot(received_df['cb_size_bytes'], bins=30, kde=True) plt.title('Distribution of Compact Block Sizes') plt.xlabel('Size (bytes)') plt.ylabel('Count') # 3. Prefill effectiveness plt.subplot(2, 2, 3) sent_df['prefill_size'] = sent_df['cb_bytes_sent'] - sent_df['cb_size_bytes'] sent_df['prefill_fit'] = sent_df['prefill_size'] <= (sent_df['tcp_window_bytes'] - (sent_df['cb_size_bytes'] % sent_df['tcp_window_bytes'])) prefill_stats = sent_df['prefill_fit'].value_counts(normalize=True) * 100 plt.bar(prefill_stats.index.map({True: 'Fit', False: 'Did Not Fit'}), prefill_stats.values) plt.title('Prefill Effectiveness') plt.ylabel('Percentage') # 4. Relationship between block size and missing bytes plt.subplot(2, 2, 4) plt.scatter(received_df['cb_size_bytes'], received_df['cb_missing_bytes'], alpha=0.5) plt.title('Block Size vs Missing Bytes') plt.xlabel('Compact Block Size (bytes)') plt.ylabel('Missing Bytes') plt.tight_layout() plt.savefig('compact_block_analysis.png', dpi=300) plt.show() # Additional visualizations plt.figure(figsize=(12, 6)) # TCP window utilization plt.subplot(1, 2, 1) sent_df['window_utilization'] = sent_df['cb_size_bytes'] / sent_df['tcp_window_bytes'] sns.histplot(sent_df['window_utilization'], bins=30, kde=True) plt.title('TCP Window Utilization') plt.xlabel('Utilization Ratio') plt.ylabel('Count') # Prefill size distribution plt.subplot(1, 2, 2) prefill_df = sent_df[sent_df['prefill_size'] > 0] sns.histplot(prefill_df['prefill_size'], bins=30, kde=True) plt.title('Prefill Size Distribution') plt.xlabel('Prefill Size (bytes)') plt.ylabel('Count') plt.tight_layout() plt.savefig('compact_block_analysis_2.png', dpi=300) plt.show()4. Update Your Main Function
def main(filepath): # Parse the logs parsed_data = parse_logs(filepath) # Analyze the data dataframes = analyze_data(parsed_data) # Create visualizations create_visualizations(dataframes) # Print summary statistics print("\nSummary Statistics:") print("=" * 50) received_df = dataframes['received_df'] sent_df = dataframes['sent_df'] # Basic statistics print(f"Total blocks received: {len(received_df)}") print(f"Reconstruction failure rate: {(received_df['cb_missing_tx_count'] > 0).mean():.2%}") # Compact block statistics print("\nCompact Block Statistics:") print(f"Average size: {received_df['cb_size_bytes'].mean():.2f} bytes") print(f"Median size: {received_df['cb_size_bytes'].median():.2f} bytes") print(f"Min size: {received_df['cb_size_bytes'].min()} bytes") print(f"Max size: {received_df['cb_size_bytes'].max()} bytes") # Prefill statistics print("\nPrefill Statistics:") prefill_df = sent_df[sent_df['prefill_size'] > 0] if not prefill_df.empty: print(f"Average prefill size: {prefill_df['prefill_size'].mean():.2f} bytes") print(f"Median prefill size: {prefill_df['prefill_size'].median():.2f} bytes") print(f"Percentage of blocks with prefill: {len(prefill_df)/len(sent_df):.2%}") else: print("No prefill data available") # TCP window statistics print("\nTCP Window Statistics:") print(f"Average window size: {sent_df['tcp_window_bytes'].mean():.2f} bytes") print(f"Median window size: {sent_df['tcp_window_bytes'].median():.2f} bytes") # Save data to CSV for further analysis received_df.to_csv('blocks_received.csv', index=False) sent_df.to_csv('blocks_sent.csv', index=False)Additional Visualization Ideas
-
Time Series Analysis: If your logs have timestamps, you could analyze trends over time.
-
Peer Analysis: Visualize which peers are most active in sending/receiving compact blocks.
-
Efficiency Metrics: Create plots showing the relationship between prefill size and reconstruction success.
-
Network Efficiency: Visualize how well the TCP windows are being utilized.
-
Block Size vs. Missing Transactions: Scatter plot showing if larger blocks tend to have more missing transactions.
Required Libraries
Make sure to install these libraries if you haven't already:
pip install pandas matplotlib seabornThis approach will give you much better insights into your compact block data with professional-looking visualizations that can help you understand patterns and relationships in the data more effectively.
-
Other models, same or similar prompt
Responses were produced in different, uncontrolled contexts (system prompts, settings, prior turns). This is not a benchmark.