can you help me customize this code for my log data """ The interface for data…
- User
can you help me customize this code for my log data """ The interface for data preprocessing.
Authors: LogPAI Team
"""
import pandas as pd import os import numpy as np import re from collections import Counter from scipy.special import expit from itertools import compress from torch.utils.data import DataLoader, Dataset
class Iterator(Dataset): def init(self, data_dict, batch_size=32, shuffle=False, num_workers=1): self.data_dict = data_dict self.keys = list(data_dict.keys()) self.iter = DataLoader(dataset=self, batch_size=batch_size, shuffle=shuffle, num_workers=num_workers)
def __getitem__(self, index): return {k: self.data_dict[k][index] for k in self.keys} def __len__(self): return self.data_dict["SessionId"].shape[0]class Vectorizer(object):
def fit_transform(self, x_train, window_y_train, y_train): self.label_mapping = {eid: idx for idx, eid in enumerate(window_y_train.unique(), 2)} self.label_mapping["#OOV"] = 0 self.label_mapping["#Pad"] = 1 self.num_labels = len(self.label_mapping) return self.transform(x_train, window_y_train, y_train) def transform(self, x, window_y, y): x["EventSequence"] = x["EventSequence"].map(lambda x: [self.label_mapping.get(item, 0) for item in x]) window_y = window_y.map(lambda x: self.label_mapping.get(x, 0)) y = y data_dict = {"SessionId": x["SessionId"].values, "window_y": window_y.values, "y": y.values, "x": np.array(x["EventSequence"].tolist())} return data_dictclass FeatureExtractor(object):
def __init__(self): self.idf_vec = None self.mean_vec = None self.events = None self.term_weighting = None self.normalization = None self.oov = None def fit_transform(self, X_seq, term_weighting=None, normalization=None, oov=False, min_count=1): """ Fit and transform the data matrix Arguments --------- X_seq: ndarray, log sequences matrix term_weighting: None or `tf-idf` normalization: None or `zero-mean` oov: bool, whether to use OOV event min_count: int, the minimal occurrence of events (default 0), only valid when oov=True. Returns ------- X_new: The transformed data matrix """ print('====== Transformed train data summary ======') self.term_weighting = term_weighting self.normalization = normalization self.oov = oov X_counts = [] for i in range(X_seq.shape[0]): event_counts = Counter(X_seq[i]) X_counts.append(event_counts) X_df = pd.DataFrame(X_counts) X_df = X_df.fillna(0) self.events = X_df.columns X = X_df.values if self.oov: oov_vec = np.zeros(X.shape[0]) if min_count > 1: idx = np.sum(X > 0, axis=0) >= min_count oov_vec = np.sum(X[:, ~idx] > 0, axis=1) X = X[:, idx] self.events = np.array(X_df.columns)[idx].tolist() X = np.hstack([X, oov_vec.reshape(X.shape[0], 1)]) num_instance, num_event = X.shape if self.term_weighting == 'tf-idf': df_vec = np.sum(X > 0, axis=0) self.idf_vec = np.log(num_instance / (df_vec + 1e-8)) idf_matrix = X * np.tile(self.idf_vec, (num_instance, 1)) X = idf_matrix if self.normalization == 'zero-mean': mean_vec = X.mean(axis=0) self.mean_vec = mean_vec.reshape(1, num_event) X = X - np.tile(self.mean_vec, (num_instance, 1)) elif self.normalization == 'sigmoid': X[X != 0] = expit(X[X != 0]) X_new = X print('Train data shape: {}-by-{}\n'.format(X_new.shape[0], X_new.shape[1])) return X_new def transform(self, X_seq): """ Transform the data matrix with trained parameters Arguments --------- X: log sequences matrix term_weighting: None or `tf-idf` Returns ------- X_new: The transformed data matrix """ print('====== Transformed test data summary ======') X_counts = [] for i in range(X_seq.shape[0]): event_counts = Counter(X_seq[i]) X_counts.append(event_counts) X_df = pd.DataFrame(X_counts) X_df = X_df.fillna(0) empty_events = set(self.events) - set(X_df.columns) for event in empty_events: X_df[event] = [0] * len(X_df) X = X_df[self.events].values if self.oov: oov_vec = np.sum(X_df[X_df.columns.difference(self.events)].values > 0, axis=1) X = np.hstack([X, oov_vec.reshape(X.shape[0], 1)]) num_instance, num_event = X.shape if self.term_weighting == 'tf-idf': idf_matrix = X * np.tile(self.idf_vec, (num_instance, 1)) X = idf_matrix if self.normalization == 'zero-mean': X = X - np.tile(self.mean_vec, (num_instance, 1)) elif self.normalization == 'sigmoid': X[X != 0] = expit(X[X != 0]) X_new = X print('Test data shape: {}-by-{}\n'.format(X_new.shape[0], X_new.shape[1])) return X_newmy log data:
frontend.head() kubernetes parent agent message url labels observer trace @timestamp data_stream service host event span transaction user_agent http 0 {'node': {'name': 'node01'}, 'pod': {'uid': '7... {'id': 'c6969009ac4370b0'} {'name': 'opentelemetry/webjs', 'version': '1.... domainLookupStart {'original': 'http://my-otel-demo-frontendprox... {'process_runtime_description': 'Web Browser',... {'hostname': 'e976764b30ab', 'type': 'apm-serv... {'id': 'acd1f6ffe7c96b533fce06443a664782'} 2024-04-24T09:21:38.608Z {'namespace': 'default', 'type': 'logs', 'data... {'node': {'name': 'node01'}, 'name': 'frontend... {'hostname': 'node01', 'name': 'node01'} {'agent_id_status': 'missing', 'ingested': '20... {'id': 'd98219ff202be5b7'} NaN NaN NaN 1 {'node': {'name': 'node01'}, 'pod': {'uid': '7... {'id': 'c6969009ac4370b0'} {'name': 'opentelemetry/webjs', 'version': '1.... domainLookupEnd {'original': 'http://my-otel-demo-frontendprox... {'process_runtime_description': 'Web Browser',... {'hostname': 'e976764b30ab', 'type': 'apm-serv... {'id': 'acd1f6ffe7c96b533fce06443a664782'} 2024-04-24T09:21:38.608Z {'namespace': 'default', 'type': 'logs', 'data... {'node': {'name': 'node01'}, 'name': 'frontend... {'hostname': 'node01', 'name': 'node01'} {'agent_id_status': 'missing', 'ingested': '20... {'id': 'd98219ff202be5b7'} NaN NaN NaN 2 {'node': {'name': 'node01'}, 'pod': {'uid': '7... {'id': 'c6969009ac4370b0'} {'name': 'opentelemetry/webjs', 'version': '1.... connectStart {'original': 'http://my-otel-demo-frontendprox... {'process_runtime_description': 'Web Browser',... {'hostname': 'e976764b30ab', 'type': 'apm-serv... {'id': 'acd1f6ffe7c96b533fce06443a664782'} 2024-04-24T09:21:38.608Z {'namespace': 'default', 'type': 'logs', 'data... {'node': {'name': 'node01'}, 'name': 'frontend... {'hostname': 'node01', 'name': 'node01'} {'agent_id_status': 'missing', 'ingested': '20... {'id': 'd98219ff202be5b7'} NaN NaN NaN 3 {'node': {'name': 'node01'}, 'pod': {'uid': '7... {'id': 'c6969009ac4370b0'} {'name': 'opentelemetry/webjs', 'version': '1.... connectEnd {'original': 'http://my-otel-demo-frontendprox... {'process_runtime_description': 'Web Browser',... {'hostname': 'e976764b30ab', 'type': 'apm-serv... {'id': 'acd1f6ffe7c96b533fce06443a664782'} 2024-04-24T09:21:38.608Z {'namespace': 'default', 'type': 'logs', 'data... {'node': {'name': 'node01'}, 'name': 'frontend... {'hostname': 'node01', 'name': 'node01'} {'agent_id_status': 'missing', 'ingested': '20... {'id': 'd98219ff202be5b7'} NaN NaN NaN 4 {'node': {'name': 'node01'}, 'pod': {'uid': '7... {'id': 'c6969009ac4370b0'} {'name': 'opentelemetry/webjs', 'version': '1.... requestStart {'original': 'http://my-otel-demo-frontendprox... {'process_runtime_description': 'Web Browser',... {'hostname': 'e976764b30ab', 'type': 'apm-serv... {'id': 'acd1f6ffe7c96b533fce06443a664782'} 2024-04-24T09:21:38.648Z {'namespace': 'default', 'type': 'logs', 'data... {'node': {'name': 'node01'}, 'name': 'frontend... {'hostname': 'node01', 'name': 'node01'} {'agent_id_status': 'missing', 'ingested': '20... {'id': 'd98219ff202be5b7'} NaN NaN NaN
- Assistant
Absolutely! To customize the existing code for your log data, you'll need to adjust the
LogParserandVectorizerclasses to fit the structure of your log data. Your log data includes various fields related to Kubernetes, observability, and timestamps, from which you may want to extract specific features or event sequences.Here’s a customized version of the original code including some modifications specific to your data:
- Log parsing: Extract relevant columns and format the event sequences.
- Vectorizing logs: Map events from the
messagecolumn to properly represent the sequences.
Here is the updated version of the code along with some helper functions:
Helper Function for Log Parsing
We'll need a specific function to parse the
messagecolumn which seems to be the event description.import pandas as pd import numpy as np from collections import Counter from scipy.special import expit from torch.utils.data import DataLoader, Dataset class LogIterator(Dataset): def __init__(self, data_dict, batch_size=32, shuffle=False, num_workers=1): self.data_dict = data_dict self.keys = list(data_dict.keys()) self.iter = DataLoader(dataset=self, batch_size=batch_size, shuffle=shuffle, num_workers=num_workers) def __getitem__(self, index): return {k: self.data_dict[k][index] for k in self.keys} def __len__(self): return self.data_dict["SessionId"].shape[0] class LogVectorizer(object): def fit_transform(self, df): self.label_mapping = {eid: idx for idx, eid in enumerate(df['message'].unique(), 2)} self.label_mapping["#OOV"] = 0 self.label_mapping["#Pad"] = 1 self.num_labels = len(self.label_mapping) return self.transform(df) def transform(self, df): df["EventSequence"] = df["message"].map(lambda x: self.label_mapping.get(x, 0)) data_dict = { "SessionId": df.index.values, "x": np.array(df["EventSequence"].tolist()) } return data_dict class LogFeatureExtractor(object): def __init__(self): self.idf_vec = None self.mean_vec = None self.events = None self.term_weighting = None self.normalization = None self.oov = None def fit_transform(self, X_seq, term_weighting=None, normalization=None, oov=False, min_count=1): print('====== Transformed train data summary ======') self.term_weighting = term_weighting self.normalization = normalization self.oov = oov X_counts = [Counter(X_seq)] X_df = pd.DataFrame(X_counts) X_df = X_df.fillna(0) self.events = X_df.columns X = X_df.values if self.oov: oov_vec = np.zeros(X.shape[0]) if min_count > 1: idx = np.sum(X > 0, axis=0) >= min_count oov_vec = np.sum(X[:, ~idx] > 0, axis=1) X = X[:, idx] self.events = np.array(X_df.columns)[idx].tolist() X = np.hstack([X, oov_vec.reshape(X.shape[0], 1)]) num_instance, num_event = X.shape if self.term_weighting == 'tf-idf': df_vec = np.sum(X > 0, axis=0) self.idf_vec = np.log(num_instance / (df_vec + 1e-8)) idf_matrix = X * np.tile(self.idf_vec, (num_instance, 1)) X = idf_matrix if self.normalization == 'zero-mean': mean_vec = X.mean(axis=0) self.mean_vec = mean_vec.reshape(1, num_event) X = X - np.tile(self.mean_vec, (num_instance, 1)) elif self.normalization == 'sigmoid': X[X != 0] = expit(X[X != 0]) X_new = X print('Train data shape: {}-by-{}\n'.format(X_new.shape[0], X_new.shape[1])) return X_new def transform(self, X_seq): print('====== Transformed test data summary ======') X_counts = [Counter(X_seq)] X_df = pd.DataFrame(X_counts) X_df = X_df.fillna(0) empty_events = set(self.events) - set(X_df.columns) for event in empty_events: X_df[event] = [0] * len(X_df) X = X_df[self.events].values if self.oov: oov_vec = np.sum(X_df[X_df.columns.difference(self.events)].values > 0, axis=1) X = np.hstack([X, oov_vec.reshape(X.shape[0], 1)]) num_instance, num_event = X.shape if self.term_weighting == 'tf-idf': idf_matrix = X * np.tile(self.idf_vec, (num_instance, 1)) X = idf_matrix if self.normalization == 'zero-mean': X = X - np.tile(self.mean_vec, (num_instance, 1)) elif self.normalization == 'sigmoid': X[X != 0] = expit(X[X != 0]) X_new = X print('Test data shape: {}-by-{}\n'.format(X_new.shape[0], X_new.shape[1])) return X_new # Assuming the data is loaded into a pandas DataFrame as `log_df` def create_event_sequence_dataframe(log_df): # Group by a session identifier and get the list of events in the session session_groups = log_df.groupby("kubernetes") sequences = session_groups["message"].apply(list).reset_index() sequences["SessionId"] = sequences.index return sequences # Example usage: log_df = pd.read_csv("your_log_file.csv") sequences_df = create_event_sequence_dataframe(log_df) vectorizer = LogVectorizer() data_dict = vectorizer.fit_transform(sequences_df) iterator = LogIterator(data_dict) for batch in iterator: print(batch)Note:
- Adjust how you group logs based on your data. Here it's grouped by
kubernetes. - The
messagefield is considered the event identifier. - Customize
LogIteratorandLogVectorizerto fit the shape and structure of your log data.
If your log data is large, you might need to read it in chunks or preprocess it to extract essential fields first.
- User
my dataframe : frontend.head() kubernetes parent agent message url labels observer trace @timestamp data_stream service host event span transaction user_agent http 0 {'node': {'name': 'node01'}, 'pod': {'uid': '7... {'id': 'c6969009ac4370b0'} {'name': 'opentelemetry/webjs', 'version': '1.... domainLookupStart {'original': 'http://my-otel-demo-frontendprox... {'process_runtime_description': 'Web Browser',... {'hostname': 'e976764b30ab', 'type': 'apm-serv... {'id': 'acd1f6ffe7c96b533fce06443a664782'} 2024-04-24T09:21:38.608Z {'namespace': 'default', 'type': 'logs', 'data... {'node': {'name': 'node01'}, 'name': 'frontend... {'hostname': 'node01', 'name': 'node01'} {'agent_id_status': 'missing', 'ingested': '20... {'id': 'd98219ff202be5b7'} NaN NaN NaN 1 {'node': {'name': 'node01'}, 'pod': {'uid': '7... {'id': 'c6969009ac4370b0'} {'name': 'opentelemetry/webjs', 'version': '1.... domainLookupEnd {'original': 'http://my-otel-demo-frontendprox... {'process_runtime_description': 'Web Browser',... {'hostname': 'e976764b30ab', 'type': 'apm-serv... {'id': 'acd1f6ffe7c96b533fce06443a664782'} 2024-04-24T09:21:38.608Z {'namespace': 'default', 'type': 'logs', 'data... {'node': {'name': 'node01'}, 'name': 'frontend... {'hostname': 'node01', 'name': 'node01'} {'agent_id_status': 'missing', 'ingested': '20... {'id': 'd98219ff202be5b7'} NaN NaN NaN 2 {'node': {'name': 'node01'}, 'pod': {'uid': '7... {'id': 'c6969009ac4370b0'} {'name': 'opentelemetry/webjs', 'version': '1.... connectStart {'original': 'http://my-otel-demo-frontendprox... {'process_runtime_description': 'Web Browser',... {'hostname': 'e976764b30ab', 'type': 'apm-serv... {'id': 'acd1f6ffe7c96b533fce06443a664782'} 2024-04-24T09:21:38.608Z {'namespace': 'default', 'type': 'logs', 'data... {'node': {'name': 'node01'}, 'name': 'frontend... {'hostname': 'node01', 'name': 'node01'} {'agent_id_status': 'missing', 'ingested': '20... {'id': 'd98219ff202be5b7'} NaN NaN NaN 3 {'node': {'name': 'node01'}, 'pod': {'uid': '7... {'id': 'c6969009ac4370b0'} {'name': 'opentelemetry/webjs', 'version': '1.... connectEnd {'original': 'http://my-otel-demo-frontendprox... {'process_runtime_description': 'Web Browser',... {'hostname': 'e976764b30ab', 'type': 'apm-serv... {'id': 'acd1f6ffe7c96b533fce06443a664782'} 2024-04-24T09:21:38.608Z {'namespace': 'default', 'type': 'logs', 'data... {'node': {'name': 'node01'}, 'name': 'frontend... {'hostname': 'node01', 'name': 'node01'} {'agent_id_status': 'missing', 'ingested': '20... {'id': 'd98219ff202be5b7'} NaN NaN NaN 4 {'node': {'name': 'node01'}, 'pod': {'uid': '7... {'id': 'c6969009ac4370b0'} {'name': 'opentelemetry/webjs', 'version': '1.... requestStart {'original': 'http://my-otel-demo-frontendprox... {'process_runtime_description': 'Web Browser',... {'hostname': 'e976764b30ab', 'type': 'apm-serv... {'id': 'acd1f6ffe7c96b533fce06443a664782'} 2024-04-24T09:21:38.648Z {'namespace': 'default', 'type': 'logs', 'data... {'node': {'name': 'node01'}, 'name': 'frontend... {'hostname': 'node01', 'name': 'node01'} {'agent_id_status': 'missing', 'ingested': '20... {'id': 'd98219ff202be5b7'} NaN NaN NaN
- Assistant
Given your dataframe, it looks like each row represents an event with a specific message. To process your log data and adjust the
VectorizerandIteratorclasses accordingly, you will want to focus on themessagecolumn to extract and sequence the events. Below is a tailored approach:- Parsing Events: We'll focus on grouping by the
kubernetesandtracecolumns as they seem to define unique sessions. - Sequencing: Use these groupings to create sequences of events.
Updated Code Implementation
import pandas as pd import numpy as np from collections import Counter from scipy.special import expit from torch.utils.data import DataLoader, Dataset class LogIterator(Dataset): def __init__(self, data_dict, batch_size=32, shuffle=False, num_workers=1): self.data_dict = data_dict self.keys = list(data_dict.keys()) self.iter = DataLoader(dataset=self, batch_size=batch_size, shuffle=shuffle, num_workers=num_workers) def __getitem__(self, index): return {k: self.data_dict[k][index] for k in self.keys} def __len__(self): return self.data_dict["SessionId"].shape[0] class LogVectorizer(object): def fit_transform(self, df): self.label_mapping = {eid: idx for idx, eid in enumerate(df['message'].unique(), 2)} self.label_mapping["#OOV"] = 0 self.label_mapping["#Pad"] = 1 self.num_labels = len(self.label_mapping) return self.transform(df) def transform(self, df): df["EventSequence"] = df["message"].map(lambda x: [self.label_mapping.get(x, 0)]) data_dict = { "SessionId": df.index.values, "x": np.array(df["EventSequence"].tolist()) } return data_dict class LogFeatureExtractor(object): def __init__(self): self.idf_vec = None self.mean_vec = None self.events = None self.term_weighting = None self.normalization = None self.oov = None def fit_transform(self, X_seq, term_weighting=None, normalization=None, oov=False, min_count=1): print('====== Transformed train data summary ======') self.term_weighting = term_weighting self.normalization = normalization self.oov = oov X_counts = [Counter(map(tuple, X_seq))] X_df = pd.DataFrame(X_counts).fillna(0) self.events = X_df.columns X = X_df.values if self.oov: oov_vec = np.zeros(X.shape[0]) if min_count > 1: idx = np.sum(X > 0, axis=0) >= min_count oov_vec = np.sum(X[:, ~idx] > 0, axis=1) X = X[:, idx] self.events = np.array(X_df.columns)[idx].tolist() X = np.hstack([X, oov_vec.reshape(X.shape[0], 1)]) num_instance, num_event = X.shape if self.term_weighting == 'tf-idf': df_vec = np.sum(X > 0, axis=0) self.idf_vec = np.log(num_instance / (df_vec + 1e-8)) idf_matrix = X * np.tile(self.idf_vec, (num_instance, 1)) X = idf_matrix if self.normalization == 'zero-mean': mean_vec = X.mean(axis=0) self.mean_vec = mean_vec.reshape(1, num_event) X = X - np.tile(self.mean_vec, (num_instance, 1)) elif self.normalization == 'sigmoid': X[X != 0] = expit(X[X != 0]) X_new = X print('Train data shape: {}-by-{}\n'.format(X_new.shape[0], X_new.shape[1])) return X_new def transform(self, X_seq): print('====== Transformed test data summary ======') X_counts = [Counter(map(tuple, X_seq))] X_df = pd.DataFrame(X_counts).fillna(0) empty_events = set(self.events) - set(X_df.columns) for event in empty_events: X_df[event] = [0] * len(X_df) X = X_df[self.events].values if self.oov: oov_vec = np.sum(X_df[X_df.columns.difference(self.events)].values > 0, axis=1) X = np.hstack([X, oov_vec.reshape(X.shape[0], 1)]) num_instance, num_event = X.shape if self.term_weighting == 'tf-idf': idf_matrix = X * np.tile(self.idf_vec, (num_instance, 1)) X = idf_matrix if self.normalization == 'zero-mean': X = X - np.tile(self.mean_vec, (num_instance, 1)) elif self.normalization == 'sigmoid': X[X != 0] = expit(X[X != 0]) X_new = X print('Test data shape: {}-by-{}\n'.format(X_new.shape[0], X_new.shape[1])) return X_new def create_event_sequence_dataframe(df): # Creating SessionId by combining kubernetes node name and pod uid df['SessionId'] = df['kubernetes'].apply(lambda x: x['node']['name']) + "-" + df['kubernetes'].apply(lambda x: x['pod']['uid']) df['Message'] = df['message'] return df[['SessionId', 'Message']] # Example usage: log_df = pd.read_csv("your_log_file.csv") # Create event sequences dataframe sequences_df = create_event_sequence_dataframe(log_df) # Vectorize the data vectorizer = LogVectorizer() data_dict = vectorizer.fit_transform(sequences_df) # Create data iterator iterator = LogIterator(data_dict) for batch in iterator: print(batch)Explanation
- LogIterator: Adapts to work with the new form of
data_dict. - LogVectorizer: Takes the event
messageand maps it accordingly, creating sequences. - LogFeatureExtractor: Prepares a more sophisticated feature extraction if needed.
Note:
- Ensure the
kubernetesandmessagecolumns are formatted correctly in your CSV. - Adjust the delimiter or parsers if your CSV/JSON is more complex.
- Parsing Events: We'll focus on grouping by the