i want to create the Interactive Cyber Threat Analysis Chatbot Using PCAP Data with…
- User
i want to create the Interactive Cyber Threat Analysis Chatbot Using PCAP Data with SLM+LLM. I want to create the customize SLM+LLM with various feature that it fast response and accurate response. add on with self-learning model
for the fast retreivel we use the RAG with fine tune + langchain for the RAG database we the chromadb with hybrid search.
for fast computation we use the thread. the code you are give give the name of file with extension.
user interaction: Website language: python host:- on the github GUI based OS:- window Limitation:- we have only 16gb RAM. NO GPU
Assistant
Interactive Cyber Threat Analysis Chatbot Using PCAP Data with SLM+LLM
This comprehensive solution combines Small Language Models (SLM) and Large Language Models (LLM) to analyze PCAP data for cybersecurity threat detection. The system leverages RAG with ChromaDB hybrid search and optimizes performance within a 16GB RAM constraint.
System Architecture
The system consists of several interconnected components:
- PCAP Processing Engine: Extracts and processes network traffic data
- Vector Database: ChromaDB with hybrid search capabilities
- Retrieval System: Implements RAG pattern with LangChain
- Model Layer: Combines SLM for fast classification and LLM for detailed analysis
- Web Interface: Flask-based GUI for user interaction
- Self-Learning Module: Feedback collection and model improvement
Core Components
PCAP Processing Engine
The PCAP processing module extracts relevant information from network traffic files.
pcap_processor.py:import pyshark import pandas as pd import threading from concurrent.futures import ThreadPoolExecutor import os class PCAPProcessor: def __init__(self, max_workers=4): self.max_workers = max_workers def process_pcap(self, pcap_file): """Process PCAP file and extract relevant data""" print(f"Processing {pcap_file}...") # Create a PyShark capture object capture = pyshark.FileCapture(pcap_file) # Initialize lists to store packet data packets_data = [] # Process each packet for packet in capture: try: packet_data = self._extract_packet_features(packet) if packet_data: packets_data.append(packet_data) except Exception as e: print(f"Error processing packet: {e}") # Convert to DataFrame df = pd.DataFrame(packets_data) # Close the capture capture.close() return df def _extract_packet_features(self, packet): """Extract relevant features from a packet""" packet_data = { 'timestamp': packet.sniff_time, 'protocol': packet.transport_layer if hasattr(packet, 'transport_layer') else packet.highest_layer, } # Extract IP information if available if hasattr(packet, 'ip'): packet_data['src_ip'] = packet.ip.src packet_data['dst_ip'] = packet.ip.dst # Extract TCP/UDP information if available if hasattr(packet, 'tcp'): packet_data['src_port'] = packet.tcp.srcport packet_data['dst_port'] = packet.tcp.dstport packet_data['flags'] = packet.tcp.flags if hasattr(packet.tcp, 'flags') else None elif hasattr(packet, 'udp'): packet_data['src_port'] = packet.udp.srcport packet_data['dst_port'] = packet.udp.dstport # Extract HTTP information if available if hasattr(packet, 'http'): if hasattr(packet.http, 'request_method'): packet_data['http_method'] = packet.http.request_method packet_data['http_uri'] = packet.http.request_uri if hasattr(packet.http, 'request_uri') else None if hasattr(packet.http, 'response_code'): packet_data['http_response_code'] = packet.http.response_code # Extract DNS information if available if hasattr(packet, 'dns'): packet_data['dns_query'] = packet.dns.qry_name if hasattr(packet.dns, 'qry_name') else None return packet_data def process_pcaps_parallel(self, pcap_dir): """Process multiple PCAP files in parallel using threads""" pcap_files = [os.path.join(pcap_dir, f) for f in os.listdir(pcap_dir) if f.endswith('.pcap')] all_data = [] with ThreadPoolExecutor(max_workers=self.max_workers) as executor: results = list(executor.map(self.process_pcap, pcap_files)) # Combine all results if results: combined_df = pd.concat(results, ignore_index=True) return combined_df return pd.DataFrame() def pcap_to_documents(self, pcap_file, chunk_size=100): """Convert PCAP data to documents for RAG""" df = self.process_pcap(pcap_file) # Group packets into conversations conversations = self._group_into_conversations(df) # Convert conversations to text documents documents = [] for i, conv in enumerate(conversations): doc_text = f"Conversation {i+1}:\n" doc_text += self._conversation_to_text(conv) documents.append({"text": doc_text, "metadata": {"source": pcap_file, "conversation_id": i+1}}) return documents def _group_into_conversations(self, df): """Group packets into conversations based on IP and port pairs""" if df.empty: return [] conversations = [] # Group by source-destination pairs if all(col in df.columns for col in ['src_ip', 'dst_ip', 'src_port', 'dst_port']): grouped = df.groupby(['src_ip', 'dst_ip', 'src_port', 'dst_port']) for name, group in grouped: conversations.append(group.sort_values('timestamp')) return conversations def _conversation_to_text(self, conversation_df): """Convert a conversation DataFrame to text format""" text = "" for idx, row in conversation_df.iterrows(): text += f"Time: {row['timestamp']}\n" text += f"Protocol: {row['protocol']}\n" text += f"Source: {row.get('src_ip', 'N/A')}:{row.get('src_port', 'N/A')}\n" text += f"Destination: {row.get('dst_ip', 'N/A')}:{row.get('dst_port', 'N/A')}\n" # Add HTTP details if available if 'http_method' in row and pd.notna(row['http_method']): text += f"HTTP Method: {row['http_method']}\n" if 'http_uri' in row and pd.notna(row['http_uri']): text += f"URI: {row['http_uri']}\n" if 'http_response_code' in row and pd.notna(row['http_response_code']): text += f"Response Code: {row['http_response_code']}\n" # Add DNS details if available if 'dns_query' in row and pd.notna(row['dns_query']): text += f"DNS Query: {row['dns_query']}\n" text += "\n" return textVector Database with ChromaDB Hybrid Search
vector_store.py:import os import chromadb from chromadb.config import Settings from langchain_community.vectorstores import Chroma from langchain_community.retrievers import BM25Retriever from langchain.retrievers import EnsembleRetriever from langchain.schema import Document from langchain_openai import OpenAIEmbeddings from langchain_community.embeddings import HuggingFaceEmbeddings import threading class VectorStore: def __init__(self, db_path="./chroma_db", embedding_model="all-MiniLM-L6-v2", use_threads=True): self.db_path = db_path self.use_threads = use_threads # Create directory if it doesn't exist os.makedirs(db_path, exist_ok=True) # Initialize embedding model if embedding_model == "openai": self.embeddings = OpenAIEmbeddings() else: # Use HuggingFace embeddings for local processing self.embeddings = HuggingFaceEmbeddings(model_name=embedding_model) # Initialize ChromaDB self.client = chromadb.PersistentClient( path=db_path, settings=Settings(allow_reset=True) ) # Create or get collection self.collection = self.client.get_or_create_collection( name="pcap_data", metadata={"hnsw:space": "cosine"} ) # Initialize Langchain Chroma wrapper self.chroma = Chroma( client=self.client, collection_name="pcap_data", embedding_function=self.embeddings ) # Lock for thread safety self.lock = threading.Lock() def add_documents(self, documents): """Add documents to the vector store""" langchain_docs = [] for doc in documents: langchain_docs.append( Document( page_content=doc["text"], metadata=doc["metadata"] ) ) with self.lock: self.chroma.add_documents(langchain_docs) def create_hybrid_retriever(self, k=5): """Create a hybrid retriever combining vector and keyword search""" # Get all documents with self.lock: raw_docs = self.chroma.get(include=["documents", "metadatas"]) # Convert to Document objects documents = [ Document(page_content=doc, metadata=meta) for doc, meta in zip(raw_docs["documents"], raw_docs["metadatas"]) ] # Create BM25 retriever bm25_retriever = BM25Retriever.from_documents(documents=documents, k=k) # Create vector search retriever similarity_search_retriever = self.chroma.as_retriever( search_type="similarity", search_kwargs={'k': k} ) # Create hybrid retriever ensemble_retriever = EnsembleRetriever( retrievers=[similarity_search_retriever, bm25_retriever], weights=[0.5, 0.5] ) return ensemble_retriever def hybrid_search(self, query, k=5): """Perform hybrid search on the vector store""" retriever = self.create_hybrid_retriever(k=k) if self.use_threads: # Execute search in a separate thread to avoid blocking search_thread = threading.Thread(target=self._threaded_search, args=(retriever, query)) search_thread.start() search_thread.join() return self.search_results else: return retriever.get_relevant_documents(query) def _threaded_search(self, retriever, query): """Execute search in a thread""" self.search_results = retriever.get_relevant_documents(query) def clear(self): """Clear the vector store""" with self.lock: self.client.reset()RAG Implementation with LangChain
rag_system.py:from langchain.chains import RetrievalQA from langchain_openai import ChatOpenAI from langchain_community.llms import HuggingFacePipeline from langchain.prompts import PromptTemplate from langchain.memory import ConversationBufferMemory import threading import transformers import torch class RAGSystem: def __init__(self, vector_store, use_openai=False, openai_api_key=None): self.vector_store = vector_store self.use_openai = use_openai # Initialize language model if use_openai and openai_api_key: self.llm = ChatOpenAI( temperature=0.1, model="gpt-3.5-turbo", openai_api_key=openai_api_key ) else: # Use a smaller local model since we have limited RAM self._initialize_local_model() # Set up conversation memory self.memory = ConversationBufferMemory( memory_key="chat_history", return_messages=True ) # Create retriever self.retriever = self.vector_store.create_hybrid_retriever() # Define prompt template template = """ You are a cybersecurity expert analyzing PCAP files. Use the following context from network traffic to answer the question. Context: {context} Question: {question} Chat History: {chat_history} Please provide a detailed analysis of the network traffic and identify any potential security issues or suspicious activities. Format your answer in a clear, structured way with sections if needed. If you cannot answer based on the given context, say so clearly. """ self.prompt = PromptTemplate( input_variables=["context", "question", "chat_history"], template=template ) # Initialize RetrievalQA chain self._initialize_qa_chain() def _initialize_local_model(self): """Initialize a local small language model""" # Using a small model that can run on CPU with limited RAM model_id = "microsoft/phi-2" # 2.7B parameters, should fit in 16GB RAM # Load model in 8-bit precision to save memory model = transformers.AutoModelForCausalLM.from_pretrained( model_id, torch_dtype=torch.float16, device_map="auto", load_in_8bit=True ) tokenizer = transformers.AutoTokenizer.from_pretrained(model_id) # Create pipeline pipe = transformers.pipeline( "text-generation", model=model, tokenizer=tokenizer, max_new_tokens=512, temperature=0.1 ) # Create LangChain wrapper self.llm = HuggingFacePipeline(pipeline=pipe) def _initialize_qa_chain(self): """Initialize the QA chain""" self.qa_chain = RetrievalQA.from_chain_type( llm=self.llm, chain_type="stuff", retriever=self.retriever, return_source_documents=True, chain_type_kwargs={ "prompt": self.prompt, "memory": self.memory } ) def process_query(self, query): """Process a user query using RAG""" # Create a thread for processing to avoid blocking process_thread = threading.Thread(target=self._threaded_process, args=(query,)) process_thread.start() process_thread.join() return self.result def _threaded_process(self, query): """Process query in a thread""" self.result = self.qa_chain({"query": query}) def add_feedback(self, query, result, feedback, is_helpful=True): """Store feedback for self-learning""" # This would be expanded in the self-learning module # For now, just store the feedback feedback_data = { "query": query, "result": result, "feedback": feedback, "is_helpful": is_helpful } # This would be used for model fine-tuning # Placeholder for actual implementation print(f"Feedback received: {feedback_data}") return feedback_dataSelf-Learning Module
self_learning.py:import json import os import pandas as pd from datetime import datetime import threading
Other models, same or similar prompt
Responses were produced in different, uncontrolled contexts (system prompts, settings, prior turns). This is not a benchmark.