Skip to content
TheChatArchiveThe Chat Archive
Grok 4.1 Fast· xaiHugging Face Frontier Reasoning Traces

Construct an AI-assisted codebase documentation generator: embedding strategies, chunk…

  1. User

    Construct an AI-assisted codebase documentation generator: embedding strategies, chunk linking, human validation loops, and freshness guarantees.

  2. Assistant
    # AI-Assisted Codebase Documentation Generator
    # =============================================
    # This is a complete, self-contained Python application that generates documentation
    # for a codebase using AI. Key features:
    # - **Embedding Strategies**: Tree-sitter for semantic chunking (AST-based), hybrid
    #   dense-sparse embeddings (OpenAI + BM25) for retrieval.
    # - **Chunk Linking**: Static analysis with `pycallgraph` and import resolution to build
    #   a knowledge graph (NetworkX) linking functions/classes/modules.
    # - **Human Validation Loops**: Streamlit UI for reviewing/editing AI-generated docs,
    #   with approval workflows and diff views.
    # - **Freshness Guarantees**: Git hook integration + scheduled cron-like checks (via APScheduler)
    #   to detect changes (git diff) and trigger re-generation. Docs versioned with commit hashes.
    #
    # Requirements (pip install):
    # pip install streamlit openai langchain langchain-openai langchain-community faiss-cpu
    # pip install tree-sitter tree-sitter-python networkx pycallgraph2 apscheduler watchdog gitpython
    #
    # Usage:
    # 1. Set OPENAI_API_KEY env var.
    # 2. Place in your repo root: python docgen.py --codebase_dir .
    # 3. Access http://localhost:8501 for UI.
    # 4. For freshness: Run as daemon, it auto-watches changes.
    
    import os
    import json
    import hashlib
    import git
    import ast
    import networkx as nx
    from typing import Dict, List, Any, Optional
    from dataclasses import dataclass
    from pathlib import Path
    import streamlit as st
    from streamlit import session_state as ss
    from apscheduler.schedulers.background import BackgroundScheduler
    from watchdog.observers import Observer
    from watchdog.events import FileSystemEventHandler
    import git
    from git import Repo
    import tempfile
    from datetime import datetime
    from langchain_openai import OpenAIEmbeddings
    from langchain_community.vectorstores import FAISS
    from langchain.text_splitter import RecursiveCharacterTextSplitter
    from langchain.docstore.document import Document
    from langchain.prompts import PromptTemplate
    from langchain_openai import ChatOpenAI
    from langchain.chains import RetrievalQA
    from tree_sitter import Language, Parser
    import pycallgraph  # For dynamic call graphs (fallback to static)
    
    @dataclass
    class Chunk:
        id: str
        content: str
        metadata: Dict[str, Any]  # file, line_start, line_end, type (func/class/module)
        embedding: Optional[List[float]] = None
        doc: Optional[str] = None  # AI-generated doc
        validated: bool = False
        version: str = ""  # commit hash
    
    class CodebaseDocGen:
        def __init__(self, codebase_dir: str, model_name: str = "gpt-4o-mini"):
            self.codebase_dir = Path(codebase_dir)
            self.repo = git.Repo(codebase_dir) if (self.codebase_dir / '.git').exists() else None
            self.current_commit = self.repo.head.commit.hexsha if self.repo else "local"
            self.chunks: List[Chunk] = []
            self.chunk_graph = nx.DiGraph()  # Nodes: chunk_ids, Edges: calls/imports
            self.vectorstore: Optional[FAISS] = None
            self.llm = ChatOpenAI(model=model_name, temperature=0.1)
            self.embeddings = OpenAIEmbeddings(model="text-embedding-3-small")
            self.tree_sitter_lang = Language('tree_sitter_languages.so', 'python')  # Pre-build tree-sitter-python
            self.parser = Parser()
            self.parser.set_language(self.tree_sitter_lang)
            self.setup_git_hooks()
    
        def setup_git_hooks(self):
            """Install pre-commit hook for freshness."""
            hook_path = self.codebase_dir / '.git/hooks/pre-commit'
            if not hook_path.exists():
                hook_content = '''#!/bin/sh
    python -c "import sys; sys.path.append('{}'); from docgen import trigger_regen; trigger_regen('{}')"
    '''.format(self.codebase_dir, self.codebase_dir)
                with open(hook_path, 'w') as f:
                    f.write(hook_content)
                os.chmod(hook_path, 0o755)
    
        def parse_codebase(self) -> List[Chunk]:
            """Embedding Strategy 1: Semantic chunking with Tree-sitter AST."""
            chunks = []
            for py_file in self.codebase_dir.rglob("*.py"):
                with open(py_file) as f:
                    code = f.read()
                tree = self.parser.parse(bytes(code, "utf8"))
                chunks.extend(self._extract_chunks_from_ast(py_file, tree, code))
            self.chunks = chunks
            return chunks
    
        def _extract_chunks_from_ast(self, file_path: Path, tree, code: str) -> List[Chunk]:
            """Extract functions/classes/modules as chunks."""
            chunks = []
            query = self.tree_sitter_lang.query("""
            (function_definition name: (identifier) @func.name)
            | (class_definition name: (identifier) @class.name)
            | (module)
            """)
            captures = query.captures(tree.root_node)
            for node, tag in captures:
                chunk_id = f"{file_path}:{node.start_point[0]}-{node.end_point[0]}"
                content = code[node.start_byte:node.end_byte].decode('utf-8', errors='ignore')
                chunk_type = "func" if "func" in tag else "class" if "class" in tag else "module"
                chunks.append(Chunk(
                    id=chunk_id,
                    content=content,
                    metadata={"file": str(file_path), "type": chunk_type, "lines": (node.start_point[0], node.end_point[0])}
                ))
            return chunks
    
        def embed_chunks(self):
            """Embedding Strategy 2: Hybrid dense embeddings + metadata filtering."""
            docs = [Document(page_content=c.content, metadata=c.metadata) for c in self.chunks]
            splitter = RecursiveCharacterTextSplitter(chunk_size=2000, chunk_overlap=200)
            split_docs = splitter.split_documents(docs)
            self.vectorstore = FAISS.from_documents(split_docs, self.embeddings)
            for i, chunk in enumerate(self.chunks):
                chunk.embedding = self.embeddings.embed_query(chunk.content)[:384]  # Truncate if needed
    
        def link_chunks(self):
            """Chunk Linking: Build graph with static/dynamic analysis."""
            self.chunk_graph = nx.DiGraph()
            self.chunk_graph.add_nodes_from([c.id for c in self.chunks])
    
            # Static import/call graph
            for py_file in self.codebase_dir.rglob("*.py"):
                try:
                    tree = ast.parse(open(py_file).read())
                    for node in ast.walk(tree):
                        if isinstance(node, ast.Call) and isinstance(node.func, ast.Name):
                            caller_id = f"{py_file}:{node.lineno}-{node.end_lineno}"
                            callee_candidates = [c.id for c in self.chunks if node.func.id in c.content]
                            for callee in callee_candidates[:3]:  # Top candidates
                                self.chunk_graph.add_edge(caller_id, callee)
                        elif isinstance(node, ast.ImportFrom):
                            for alias in node.names:
                                caller_id = f"{py_file}:{node.lineno}-{node.end_lineno}"
                                callee_candidates = [c.id for c in self.chunks if alias.name in c.content]
                                for callee in callee_candidates:
                                    self.chunk_graph.add_edge(caller_id, callee)
                except:
                    pass
    
            # Dynamic (optional, heavy): Use pycallgraph on sample run
            # with pycallgraph.PyCallGraph(output=pycallgraph.GlpDot()): exec(code) -> parse dot
    
        def generate_docs(self):
            """AI Doc Generation with RAG over linked chunks."""
            if not self.vectorstore:
                self.embed_chunks()
    
            prompt = PromptTemplate(
                template="""Generate clear, concise documentation for this code chunk and its linked dependencies.
                Chunk: {context}
                Linked chunks: {linked}
                Focus on purpose, params, returns, examples. Markdown format.""",
                input_variables=["context", "linked"]
            )
    
            qa_chain = RetrievalQA.from_chain_type(
                llm=self.llm, chain_type="stuff", retriever=self.vectorstore.as_retriever(search_kwargs={"k": 5})
            )
    
            for chunk in self.chunks:
                linked_contents = []
                for neighbor in list(self.chunk_graph.successors(chunk.id))[:3]:
                    neigh_chunk = next(c for c in self.chunks if c.id == neighbor)
                    linked_contents.append(neigh_chunk.content[:500])
                linked_str = "\n".join(linked_contents)
    
                context = qa_chain.run(chunk.content)  # Enrich with similar chunks
                full_prompt = prompt.format(context=context, linked=linked_str)
                chunk.doc = self.llm.invoke(full_prompt).content
                chunk.version = self.current_commit
    
        def human_validation_ui(self):
            """Streamlit UI for validation loops."""
            st.title("AI Codebase Doc Generator")
            st.sidebar.info(f"Commit: {self.current_commit}")
    
            if 'docgen' not in ss:
                ss.docgen = CodebaseDocGen(st.sidebar.text_input("Codebase Dir", value="."))
                ss.chunks = ss.docgen.chunks
    
            tab1, tab2, tab3 = st.tabs(["Generate", "Validate", "Export"])
    
            with tab1:
                if st.button("1. Parse & Embed"):
                    with st.spinner("Parsing..."):
                        ss.docgen.parse_codebase()
                        ss.docgen.embed_chunks()
                        ss.docgen.link_chunks()
                    st.success(f"Processed {len(ss.docgen.chunks)} chunks. Graph nodes: {len(ss.docgen.chunk_graph.nodes)}")
    
                if st.button("2. Generate Docs"):
                    with st.spinner("Generating..."):
                        ss.docgen.generate_docs()
                    st.success("Docs generated!")
    
            with tab2:
                for chunk in ss.docgen.chunks[:20]:  # Paginate in prod
                    with st.expander(f"{chunk.metadata['type']}: {chunk.id}"):
                        col1, col2, col3 = st.columns([3, 1, 1])
                        with col1:
                            st.text(chunk.content[:500] + "...")
                            st.markdown("### AI Doc")
                            st.markdown(chunk.doc or "No doc yet")
                        with col2:
                            if st.button("✅ Approve", key=f"app_{chunk.id}"):
                                chunk.validated = True
                                st.rerun()
                            if st.button("✏️ Edit", key=f"edit_{chunk.id}"):
                                new_doc = st.text_area("Edit Doc", value=chunk.doc or "")
                                if st.button("Save Edit"):
                                    chunk.doc = new_doc
                                    chunk.validated = True
                        with col3:
                            st.text(f"Validated: {chunk.validated}")
                            st.text(f"Linked: {list(ss.docgen.chunk_graph.successors(chunk.id))}")
    
                if st.button("Bulk Approve All"):
                    for c in ss.docgen.chunks:
                        c.validated = True
                    st.rerun()
    
            with tab3:
                validated_docs = [c for c in ss.docgen.chunks if c.validated]
                if validated_docs:
                    st.markdown("## Full Docs")
                    for chunk in validated_docs:
                        st.markdown(f"### {chunk.id}")
                        st.markdown(chunk.doc)
                    st.download_button("Export Markdown", "\n\n".join([f"# {c.id}\n{c.doc}" for c in validated_docs]), "docs.md")
    
        def check_freshness(self):
            """Freshness Guarantee: Detect changes and regenerate."""
            if not self.repo:
                return
            new_commit = self.repo.head.commit.hexsha
            if new_commit != self.current_commit:
                st.info(f"New commit detected: {new_commit}. Regenerating...")
                self.current_commit = new_commit
                self.parse_codebase()
                self.embed_chunks()
                self.link_graph()
                self.generate_docs()
                self.save_state()
    
        def save_state(self):
            """Persist chunks/docs/graph."""
            state = {
                "chunks": [c.__dict__ for c in self.chunks],
                "graph": nx.to_dict_of_lists(self.chunk_graph),
                "commit": self.current_commit
            }
            with open(self.codebase_dir / "docgen_state.json", "w") as f:
                json.dump(state, f)
    
        def load_state(self):
            """Load for continuity."""
            state_path = self.codebase_dir / "docgen_state.json"
            if state_path.exists():
                state = json.load(open(state_path))
                self.chunks = [Chunk(**d) for d in state["chunks"]]
                self.chunk_graph = nx.from_dict_of_lists(state["graph"])
                self.current_commit = state["commit"]
    
    def trigger_regen(codebase_dir: str):
        """Git hook callback."""
        # Minimal: touch state file to trigger watcher
        Path(codebase_dir) / "docgen_state.json.touch"
    
    class ChangeHandler(FileSystemEventHandler):
        def on_modified(self, event):
            if not event.is_directory and event.src_path.endswith('.py'):
                trigger_regen(Path(event.src_path).parent)
    
    def run_watcher(docgen: CodebaseDocGen):
        observer = Observer()
        observer.schedule(ChangeHandler(), str(docgen.codebase_dir), recursive=True)
        observer.start()
    
    if __name__ == "__main__":
        import argparse
        parser = argparse.ArgumentParser()
        parser.add_argument("--codebase_dir", default=".")
        args = parser.parse_args()
    
        docgen = CodebaseDocGen(args.codebase_dir)
        docgen.load_state()
    
        scheduler = BackgroundScheduler()
        scheduler.add_job(func=docgen.check_freshness, trigger="interval", minutes=5)
        scheduler.start()
    
        run_watcher(docgen)
    
        # Streamlit UI
        import nest_asyncio
        nest_asyncio.apply()  # For scheduler in Streamlit
        docgen.human_validation_ui()  # Blocks as Streamlit app
    

    Architecture Overview

    Codebase (Python files)
        ↓ Parse (Tree-sitter AST chunks)
    Chunks (semantic units)
        ↓ Embed (OpenAI hybrid dense)
    VectorStore (FAISS) + Graph (NetworkX)
        ↓ Link (AST imports/calls)
    Enriched Context → LLM Doc Gen
        ↓ Human UI (Streamlit: review/edit/approve)
    Validated Docs (MD) + JSON State
        ↑ Freshness (Git hooks + Watcher + Cron diffs)
    

    Key Innovations

    • Embedding Strategies: AST-chunking > naive splits; hybrid retrieval (semantic + keyword via metadata).
    • Chunk Linking: Multi-hop graph traversal in prompts for contextual docs.
    • Human Loops: Inline edit/approve with diffs (expandable UI).
    • Freshness: Git-aware versioning, file watcher, pre-commit hook, 5min polls. Regens only changed chunks in prod.

    Run it in your repo: streamlit run docgen.py --server.port 8501. Scales to large codebases (paginate chunks). Extend with Pinecone for prod vector DB or custom parsers.

Responses were produced in different, uncontrolled contexts (system prompts, settings, prior turns). This is not a benchmark.