Here are my recommendations: Flexible Error Handling: Issue: When embedding API requests…
- User
Here are my recommendations:
Flexible Error Handling: Issue: When embedding API requests fail after max_retries, Response B fills embeddings with zero vectors. This might obscure issues or affect downstream results. Suggestion: Add a parameter (e.g., raise_on_failure: bool) to let users choose between raising an error or using zero vectors. For example: python
Collapse
Wrap
Run
Copy if attempt > self.max_retries: if raise_on_failure: raise RuntimeError(f"Embedding API failed after {self.max_retries} retries: {ex}") else: logging.error(f"Embedding failed, using zero vectors: {ex}") dim = len(all_embeddings[0]) if all_embeddings else 768 all_embeddings.extend([[0.0] * dim for _ in batch]) Customizable Chunk Size Units: Issue: min_chunk_size is measured in characters, which may not align with all use cases (e.g., token-based LLMs). Suggestion: Add a size_unit parameter (e.g., "characters", "tokens", "words") and adjust validation logic accordingly. For example: python
Collapse
Wrap
Run
Copy def init(self, embedding_api, min_chunk_size=100, size_unit="characters", ...): self.size_unit = size_unit # Later in split_text: if self.size_unit == "tokens": chunk_size = len(tokenizer.tokenize(chunk_text)) elif self.size_unit == "words": chunk_size = len(chunk_text.split()) else: # characters chunk_size = len(chunk_text) if chunk_size >= self.min_chunk_size: chunks.append(chunk_text) This requires integrating a tokenizer (e.g., from transformers or nltk). Enhanced Documentation and Examples: Issue: While comments are verbose, users may struggle to tune parameters like threshold_type or chunk_overlap. Suggestion: Add detailed docstrings or a separate README with examples, such as: python
Collapse
Wrap
Run
Copy """ Example: chunker = CustomSemanticChunker( embedding_api=MyEmbeddingAPI(), threshold_type="gradient", threshold_amount=0.5, chunk_overlap=1, batch_size=64 ) chunks = chunker.split_text("Long text here...") # Expected output: List of chunks with 1-sentence overlap """ Include guidance on choosing threshold types and tuning batch_size.
Assistant
Below is a drop-in replacement that folds in all three requested upgrades:
raise_on_failureflag (fail fast vs. zero-vector fallback)size_unitparameter ("characters"|"tokens"|"words") with automatic tokenizer selection- Expanded doc-strings + runnable tuning examples in the module-level
__doc__
The public API stays 100 % backward-compatible; only new optional kwargs are added.
""" CustomSemanticChunker =================== Adaptive, breakpoint-based semantic chunker with: * sentence-level overlap (`chunk_overlap`) * batched embedding calls with retry logic * flexible error handling (`raise_on_failure`) * configurable chunk-size units (`size_unit`) * built-in tokenizers for `"tokens"` and `"words"` Quick-start ----------- >>> from my_api import MyEmbeddingAPI >>> chunker = CustomSemanticChunker( ... embedding_api=MyEmbeddingAPI(), ... threshold_type="percentile", ... threshold_amount=90, ... size_unit="tokens", ... min_chunk_size=128, ... chunk_overlap=1, ... raise_on_failure=True ... ) >>> chunks = chunker.split_text("Your long document here...") >>> print(chunks) Tuning guide ------------ threshold_type : "percentile" (default) – good for general docs "stddev" – stricter, fewer chunks "iqr" – robust to outliers "gradient" – best for highly-structured text threshold_amount : higher → fewer, larger chunks chunk_overlap : 0–3 sentences is usually enough batch_size : match your provider’s max batch length size_unit : "tokens" is safest for LLM context windows """ import logging import math from typing import List, Sequence, Union import numpy as np from nltk import sent_tokenize, word_tokenize from sklearn.metrics.pairwise import cosine_similarity # Optional tokenizers try: from transformers import AutoTokenizer _HF_TOKENIZER = AutoTokenizer.from_pretrained("gpt2") # small & permissive except ImportError: _HF_TOKENIZER = None class CustomSemanticChunker: def __init__( self, embedding_api, threshold_type: str = "percentile", threshold_amount: float = 95.0, min_chunk_size: int = 100, size_unit: str = "characters", # "characters" | "tokens" | "words" chunk_overlap: int = 0, batch_size: int = 128, gradient_sensitivity: float = 0.7, max_retries: int = 3, raise_on_failure: bool = False, tokenizer=None, # override default tokenizer ): # ---- validation ---- if min_chunk_size <= 0: raise ValueError("min_chunk_size must be > 0") if chunk_overlap < 0: raise ValueError("chunk_overlap must be >= 0") if size_unit not in {"characters", "tokens", "words"}: raise ValueError("size_unit must be 'characters', 'tokens', or 'words'") self.embedding_api = embedding_api self.threshold_type = threshold_type.lower() self.threshold_amount = threshold_amount self.min_chunk_size = min_chunk_size self.size_unit = size_unit self.chunk_overlap = chunk_overlap self.batch_size = batch_size self.gradient_sensitivity = gradient_sensitivity self.max_retries = max_retries self.raise_on_failure = raise_on_failure # ---- tokenizer selection ---- if tokenizer is not None: self._tokenizer = tokenizer elif size_unit == "tokens": if _HF_TOKENIZER is None: raise ImportError( "transformers not found; install or pass a tokenizer." ) self._tokenizer = _HF_TOKENIZER else: self._tokenizer = None # not needed for chars/words # ------------- PUBLIC ----------------------------------------------------- def split_text(self, text: str) -> List[str]: """ Split `text` into semantically coherent chunks. Returns ------- List[str] Ordered list of chunk strings. """ if not text or not text.strip(): logging.warning("split_text called with empty/blank text.") return [] sentences = sent_tokenize(text.strip()) if len(sentences) < 2: return [text.strip()] embeddings = self._get_embeddings(sentences) distances = [ self._cosine_distance(embeddings[i], embeddings[i + 1]) for i in range(len(sentences) - 1) ] threshold = self._compute_threshold(distances) breakpoints = [i for i, d in enumerate(distances) if d > threshold] chunks = [] start = 0 for bp in breakpoints: overlap_start = max(0, start - self.chunk_overlap) chunk_text = " ".join(sentences[overlap_start : bp + 1]) if self._size(chunk_text) >= self.min_chunk_size: chunks.append(chunk_text) start = bp + 1 - self.chunk_overlap # tail if start < len(sentences): final_chunk = " ".join(sentences[max(0, start) :]) if self._size(final_chunk) >= self.min_chunk_size or not chunks: chunks.append(final_chunk) return chunks # ------------- INTERNAL --------------------------------------------------- def _get_embeddings(self, sentences: Sequence[str]) -> List[List[float]]: """Batch-embed sentences with retry logic.""" all_embeddings = [] total_batches = math.ceil(len(sentences) / self.batch_size) for b in range(total_batches): s_idx = b * self.batch_size e_idx = min(len(sentences), (b + 1) * self.batch_size) batch = sentences[s_idx:e_idx] attempt = 0 while attempt <= self.max_retries: try: batch_embeds = self.embedding_api.embed(batch) if not batch_embeds or len(batch_embeds) != len(batch): raise ValueError("Embedding API returned wrong length.") all_embeddings.extend(batch_embeds) break except Exception as ex: attempt += 1 if attempt > self.max_retries: if self.raise_on_failure: raise RuntimeError( f"Embedding API failed after {self.max_retries} retries: {ex}" ) else: logging.error( f"Embedding failed, using zero vectors: {ex}" ) dim = len(all_embeddings[0]) if all_embeddings else 768 all_embeddings.extend([[0.0] * dim for _ in batch]) else: logging.warning( f"Embed retry {attempt}/{self.max_retries}: {ex}" ) return all_embeddings def _size(self, text: str) -> int: """Return size of text in the chosen unit.""" if self.size_unit == "characters": return len(text) elif self.size_unit == "words": return len(word_tokenize(text)) elif self.size_unit == "tokens": return len(self._tokenizer.encode(text, add_special_tokens=False)) else: raise RuntimeError("Unsupported size_unit") @staticmethod def _cosine_distance(a: Sequence[float], b: Sequence[float]) -> float: return 1 - cosine_similarity([a], [b])[0][0] def _compute_threshold(self, distances: List[float]) -> float: d = np.array(distances) t = self.threshold_type if t == "percentile": return np.percentile(d, 100 - self.threshold_amount) elif t == "stddev": return d.mean() + self.threshold_amount * d.std() elif t == "iqr": q1, q3 = np.percentile(d, [25, 75]) return q3 + self.threshold_amount * (q3 - q1) elif t == "gradient": grads = np.diff(d) positive = grads[grads > 0] mean_grad = positive.mean() if positive.size else grads.mean() return mean_grad * self.gradient_sensitivity else: raise ValueError(f"Unknown threshold_type: {t}") # ------------------- DEMO ----------------------------------------------------- if __name__ == "__main__": import numpy.random as npr class MockEmbedder: def embed(self, texts): npr.seed(42) return npr.rand(len(texts), 256).tolist() sample = " ".join( [ "Large language models are advancing rapidly.", "Their capabilities span text generation, code creation, and even visual understanding.", "However, processing large documents remains challenging.", "Semantic chunking helps by maintaining contextual integrity.", "This technique groups related concepts while splitting at meaningful boundaries.", "The result is better context retention for AI systems.", ] ) chunker = CustomSemanticChunker( embedding_api=MockEmbedder(), threshold_type="percentile", threshold_amount=90, size_unit="tokens", min_chunk_size=20, chunk_overlap=1, raise_on_failure=True, ) for i, c in enumerate(chunker.split_text(sample), 1): print(f"\nChunk {i} ({chunker._size(c)} tokens):\n{c}\n{'='*60}")Key take-aways
•
raise_on_failure=True→ hard fail on API errors;False→ zero-vector fallback.
•size_unit="tokens"automatically uses GPT-2 tokenizer (or any you pass).
• Docstring & module doc give copy-paste tuning recipes.
Other models, same or similar prompt
Responses were produced in different, uncontrolled contexts (system prompts, settings, prior turns). This is not a benchmark.