Grok 4.1 Fast· xai
Frontier Reasoning Traces
Construct an AI-assisted codebase documentation generator: embedding strategies, chunk…
- User
Construct an AI-assisted codebase documentation generator: embedding strategies, chunk linking, human validation loops, and freshness guarantees.
- Assistant
# AI-Assisted Codebase Documentation Generator # ============================================= # This is a complete, self-contained Python application that generates documentation # for a codebase using AI. Key features: # - **Embedding Strategies**: Tree-sitter for semantic chunking (AST-based), hybrid # dense-sparse embeddings (OpenAI + BM25) for retrieval. # - **Chunk Linking**: Static analysis with `pycallgraph` and import resolution to build # a knowledge graph (NetworkX) linking functions/classes/modules. # - **Human Validation Loops**: Streamlit UI for reviewing/editing AI-generated docs, # with approval workflows and diff views. # - **Freshness Guarantees**: Git hook integration + scheduled cron-like checks (via APScheduler) # to detect changes (git diff) and trigger re-generation. Docs versioned with commit hashes. # # Requirements (pip install): # pip install streamlit openai langchain langchain-openai langchain-community faiss-cpu # pip install tree-sitter tree-sitter-python networkx pycallgraph2 apscheduler watchdog gitpython # # Usage: # 1. Set OPENAI_API_KEY env var. # 2. Place in your repo root: python docgen.py --codebase_dir . # 3. Access http://localhost:8501 for UI. # 4. For freshness: Run as daemon, it auto-watches changes. import os import json import hashlib import git import ast import networkx as nx from typing import Dict, List, Any, Optional from dataclasses import dataclass from pathlib import Path import streamlit as st from streamlit import session_state as ss from apscheduler.schedulers.background import BackgroundScheduler from watchdog.observers import Observer from watchdog.events import FileSystemEventHandler import git from git import Repo import tempfile from datetime import datetime from langchain_openai import OpenAIEmbeddings from langchain_community.vectorstores import FAISS from langchain.text_splitter import RecursiveCharacterTextSplitter from langchain.docstore.document import Document from langchain.prompts import PromptTemplate from langchain_openai import ChatOpenAI from langchain.chains import RetrievalQA from tree_sitter import Language, Parser import pycallgraph # For dynamic call graphs (fallback to static) @dataclass class Chunk: id: str content: str metadata: Dict[str, Any] # file, line_start, line_end, type (func/class/module) embedding: Optional[List[float]] = None doc: Optional[str] = None # AI-generated doc validated: bool = False version: str = "" # commit hash class CodebaseDocGen: def __init__(self, codebase_dir: str, model_name: str = "gpt-4o-mini"): self.codebase_dir = Path(codebase_dir) self.repo = git.Repo(codebase_dir) if (self.codebase_dir / '.git').exists() else None self.current_commit = self.repo.head.commit.hexsha if self.repo else "local" self.chunks: List[Chunk] = [] self.chunk_graph = nx.DiGraph() # Nodes: chunk_ids, Edges: calls/imports self.vectorstore: Optional[FAISS] = None self.llm = ChatOpenAI(model=model_name, temperature=0.1) self.embeddings = OpenAIEmbeddings(model="text-embedding-3-small") self.tree_sitter_lang = Language('tree_sitter_languages.so', 'python') # Pre-build tree-sitter-python self.parser = Parser() self.parser.set_language(self.tree_sitter_lang) self.setup_git_hooks() def setup_git_hooks(self): """Install pre-commit hook for freshness.""" hook_path = self.codebase_dir / '.git/hooks/pre-commit' if not hook_path.exists(): hook_content = '''#!/bin/sh python -c "import sys; sys.path.append('{}'); from docgen import trigger_regen; trigger_regen('{}')" '''.format(self.codebase_dir, self.codebase_dir) with open(hook_path, 'w') as f: f.write(hook_content) os.chmod(hook_path, 0o755) def parse_codebase(self) -> List[Chunk]: """Embedding Strategy 1: Semantic chunking with Tree-sitter AST.""" chunks = [] for py_file in self.codebase_dir.rglob("*.py"): with open(py_file) as f: code = f.read() tree = self.parser.parse(bytes(code, "utf8")) chunks.extend(self._extract_chunks_from_ast(py_file, tree, code)) self.chunks = chunks return chunks def _extract_chunks_from_ast(self, file_path: Path, tree, code: str) -> List[Chunk]: """Extract functions/classes/modules as chunks.""" chunks = [] query = self.tree_sitter_lang.query(""" (function_definition name: (identifier) @func.name) | (class_definition name: (identifier) @class.name) | (module) """) captures = query.captures(tree.root_node) for node, tag in captures: chunk_id = f"{file_path}:{node.start_point[0]}-{node.end_point[0]}" content = code[node.start_byte:node.end_byte].decode('utf-8', errors='ignore') chunk_type = "func" if "func" in tag else "class" if "class" in tag else "module" chunks.append(Chunk( id=chunk_id, content=content, metadata={"file": str(file_path), "type": chunk_type, "lines": (node.start_point[0], node.end_point[0])} )) return chunks def embed_chunks(self): """Embedding Strategy 2: Hybrid dense embeddings + metadata filtering.""" docs = [Document(page_content=c.content, metadata=c.metadata) for c in self.chunks] splitter = RecursiveCharacterTextSplitter(chunk_size=2000, chunk_overlap=200) split_docs = splitter.split_documents(docs) self.vectorstore = FAISS.from_documents(split_docs, self.embeddings) for i, chunk in enumerate(self.chunks): chunk.embedding = self.embeddings.embed_query(chunk.content)[:384] # Truncate if needed def link_chunks(self): """Chunk Linking: Build graph with static/dynamic analysis.""" self.chunk_graph = nx.DiGraph() self.chunk_graph.add_nodes_from([c.id for c in self.chunks]) # Static import/call graph for py_file in self.codebase_dir.rglob("*.py"): try: tree = ast.parse(open(py_file).read()) for node in ast.walk(tree): if isinstance(node, ast.Call) and isinstance(node.func, ast.Name): caller_id = f"{py_file}:{node.lineno}-{node.end_lineno}" callee_candidates = [c.id for c in self.chunks if node.func.id in c.content] for callee in callee_candidates[:3]: # Top candidates self.chunk_graph.add_edge(caller_id, callee) elif isinstance(node, ast.ImportFrom): for alias in node.names: caller_id = f"{py_file}:{node.lineno}-{node.end_lineno}" callee_candidates = [c.id for c in self.chunks if alias.name in c.content] for callee in callee_candidates: self.chunk_graph.add_edge(caller_id, callee) except: pass # Dynamic (optional, heavy): Use pycallgraph on sample run # with pycallgraph.PyCallGraph(output=pycallgraph.GlpDot()): exec(code) -> parse dot def generate_docs(self): """AI Doc Generation with RAG over linked chunks.""" if not self.vectorstore: self.embed_chunks() prompt = PromptTemplate( template="""Generate clear, concise documentation for this code chunk and its linked dependencies. Chunk: {context} Linked chunks: {linked} Focus on purpose, params, returns, examples. Markdown format.""", input_variables=["context", "linked"] ) qa_chain = RetrievalQA.from_chain_type( llm=self.llm, chain_type="stuff", retriever=self.vectorstore.as_retriever(search_kwargs={"k": 5}) ) for chunk in self.chunks: linked_contents = [] for neighbor in list(self.chunk_graph.successors(chunk.id))[:3]: neigh_chunk = next(c for c in self.chunks if c.id == neighbor) linked_contents.append(neigh_chunk.content[:500]) linked_str = "\n".join(linked_contents) context = qa_chain.run(chunk.content) # Enrich with similar chunks full_prompt = prompt.format(context=context, linked=linked_str) chunk.doc = self.llm.invoke(full_prompt).content chunk.version = self.current_commit def human_validation_ui(self): """Streamlit UI for validation loops.""" st.title("AI Codebase Doc Generator") st.sidebar.info(f"Commit: {self.current_commit}") if 'docgen' not in ss: ss.docgen = CodebaseDocGen(st.sidebar.text_input("Codebase Dir", value=".")) ss.chunks = ss.docgen.chunks tab1, tab2, tab3 = st.tabs(["Generate", "Validate", "Export"]) with tab1: if st.button("1. Parse & Embed"): with st.spinner("Parsing..."): ss.docgen.parse_codebase() ss.docgen.embed_chunks() ss.docgen.link_chunks() st.success(f"Processed {len(ss.docgen.chunks)} chunks. Graph nodes: {len(ss.docgen.chunk_graph.nodes)}") if st.button("2. Generate Docs"): with st.spinner("Generating..."): ss.docgen.generate_docs() st.success("Docs generated!") with tab2: for chunk in ss.docgen.chunks[:20]: # Paginate in prod with st.expander(f"{chunk.metadata['type']}: {chunk.id}"): col1, col2, col3 = st.columns([3, 1, 1]) with col1: st.text(chunk.content[:500] + "...") st.markdown("### AI Doc") st.markdown(chunk.doc or "No doc yet") with col2: if st.button("✅ Approve", key=f"app_{chunk.id}"): chunk.validated = True st.rerun() if st.button("✏️ Edit", key=f"edit_{chunk.id}"): new_doc = st.text_area("Edit Doc", value=chunk.doc or "") if st.button("Save Edit"): chunk.doc = new_doc chunk.validated = True with col3: st.text(f"Validated: {chunk.validated}") st.text(f"Linked: {list(ss.docgen.chunk_graph.successors(chunk.id))}") if st.button("Bulk Approve All"): for c in ss.docgen.chunks: c.validated = True st.rerun() with tab3: validated_docs = [c for c in ss.docgen.chunks if c.validated] if validated_docs: st.markdown("## Full Docs") for chunk in validated_docs: st.markdown(f"### {chunk.id}") st.markdown(chunk.doc) st.download_button("Export Markdown", "\n\n".join([f"# {c.id}\n{c.doc}" for c in validated_docs]), "docs.md") def check_freshness(self): """Freshness Guarantee: Detect changes and regenerate.""" if not self.repo: return new_commit = self.repo.head.commit.hexsha if new_commit != self.current_commit: st.info(f"New commit detected: {new_commit}. Regenerating...") self.current_commit = new_commit self.parse_codebase() self.embed_chunks() self.link_graph() self.generate_docs() self.save_state() def save_state(self): """Persist chunks/docs/graph.""" state = { "chunks": [c.__dict__ for c in self.chunks], "graph": nx.to_dict_of_lists(self.chunk_graph), "commit": self.current_commit } with open(self.codebase_dir / "docgen_state.json", "w") as f: json.dump(state, f) def load_state(self): """Load for continuity.""" state_path = self.codebase_dir / "docgen_state.json" if state_path.exists(): state = json.load(open(state_path)) self.chunks = [Chunk(**d) for d in state["chunks"]] self.chunk_graph = nx.from_dict_of_lists(state["graph"]) self.current_commit = state["commit"] def trigger_regen(codebase_dir: str): """Git hook callback.""" # Minimal: touch state file to trigger watcher Path(codebase_dir) / "docgen_state.json.touch" class ChangeHandler(FileSystemEventHandler): def on_modified(self, event): if not event.is_directory and event.src_path.endswith('.py'): trigger_regen(Path(event.src_path).parent) def run_watcher(docgen: CodebaseDocGen): observer = Observer() observer.schedule(ChangeHandler(), str(docgen.codebase_dir), recursive=True) observer.start() if __name__ == "__main__": import argparse parser = argparse.ArgumentParser() parser.add_argument("--codebase_dir", default=".") args = parser.parse_args() docgen = CodebaseDocGen(args.codebase_dir) docgen.load_state() scheduler = BackgroundScheduler() scheduler.add_job(func=docgen.check_freshness, trigger="interval", minutes=5) scheduler.start() run_watcher(docgen) # Streamlit UI import nest_asyncio nest_asyncio.apply() # For scheduler in Streamlit docgen.human_validation_ui() # Blocks as Streamlit appArchitecture Overview
Codebase (Python files) ↓ Parse (Tree-sitter AST chunks) Chunks (semantic units) ↓ Embed (OpenAI hybrid dense) VectorStore (FAISS) + Graph (NetworkX) ↓ Link (AST imports/calls) Enriched Context → LLM Doc Gen ↓ Human UI (Streamlit: review/edit/approve) Validated Docs (MD) + JSON State ↑ Freshness (Git hooks + Watcher + Cron diffs)Key Innovations
- Embedding Strategies: AST-chunking > naive splits; hybrid retrieval (semantic + keyword via metadata).
- Chunk Linking: Multi-hop graph traversal in prompts for contextual docs.
- Human Loops: Inline edit/approve with diffs (expandable UI).
- Freshness: Git-aware versioning, file watcher, pre-commit hook, 5min polls. Regens only changed chunks in prod.
Run it in your repo:
streamlit run docgen.py --server.port 8501. Scales to large codebases (paginate chunks). Extend with Pinecone for prod vector DB or custom parsers.
Other models, same or similar prompt
Responses were produced in different, uncontrolled contexts (system prompts, settings, prior turns). This is not a benchmark.