Grok 4.1 Fast· xai
Frontier Reasoning Traces
Build a synthetic monitoring suite: golden traces and causality checks.
- User
Build a synthetic monitoring suite: golden traces and causality checks.
- Assistant
#!/usr/bin/env python3 """ Synthetic Monitoring Suite: Golden Traces & Causality Checks =========================================================== This suite provides a framework for synthetic monitoring in distributed systems. It uses OpenTelemetry-inspired trace models (spans with attributes, events, links). Key Components: - **Golden Traces**: Pre-defined exemplary traces representing "happy path" behaviors. These are synthetic baselines for anomaly detection. - **Causality Checks**: Rule-based validation of causal relationships (e.g., span ordering, parent-child hierarchies, duration SLAs, attribute propagation, event sequencing). - **Monitoring Engine**: Compares runtime traces against golden traces, scoring deviations and flagging causality violations. Usage: 1. Define golden traces for your services. 2. Ingest runtime traces (e.g., from OpenTelemetry exporter). 3. Run checks and get alerts/reports. Extensible for production: Integrate with Prometheus, Grafana, or Kafka for real-time monitoring. """ import json import uuid from dataclasses import dataclass, asdict from typing import List, Dict, Any, Optional, Tuple from enum import Enum from datetime import datetime, timedelta import hashlib class SpanKind(Enum): INTERNAL = "INTERNAL" SERVER = "SERVER" CLIENT = "CLIENT" PRODUCER = "PRODUCER" CONSUMER = "CONSUMER" @dataclass class Span: trace_id: str span_id: str parent_span_id: Optional[str] name: str kind: SpanKind start_time: int # Unix ns end_time: int # Unix ns attributes: Dict[str, Any] events: List[Dict[str, Any]] # timestamped events status: Dict[str, Any] # e.g., {"code": "OK"} links: List[Dict[str, Any]] @dataclass class Trace: trace_id: str spans: List[Span] resource: Dict[str, Any] # e.g., service name, version class CausalityViolation(Exception): pass class CausalityRule: """Defines a causal invariant (e.g., 'auth span must precede db query').""" def __init__(self, name: str, condition: callable): self.name = name self.condition = condition # fn(trace: Trace) -> bool or raises CausalityViolation def check(self, trace: Trace) -> bool: try: self.condition(trace) return True except CausalityViolation: return False class GoldenTraceMatcher: """Fuzzy matcher for golden traces with tolerance for minor variations.""" def __init__(self, golden_trace: Trace, tolerance_attrs: float = 0.8, tolerance_duration: float = 0.2): self.golden = golden_trace self.tolerance_attrs = tolerance_attrs # 80% attr match self.tolerance_duration = tolerance_duration # ±20% duration def score_match(self, candidate: Trace) -> Tuple[float, List[str]]: """Returns match score (0-1) and deviation reasons.""" if candidate.trace_id != self.golden.trace_id: return 0.0, ["trace_id mismatch"] golden_spans_by_name = {s.name: s for s in self.golden.spans} candidate_spans_by_name = {s.name: s for s in candidate.spans} deviations = [] total_score = 0.0 span_count = 0 for name, golden_span in golden_spans_by_name.items(): if name not in candidate_spans_by_name: deviations.append(f"missing span: {name}") continue cand_span = candidate_spans_by_name[name] span_score = self._score_span(golden_span, cand_span) total_score += span_score span_count += 1 if span_score < 0.7: deviations.append(f"low score on {name}: {span_score}") avg_score = total_score / max(span_count, 1) return avg_score, deviations def _score_span(self, golden: Span, cand: Span) -> float: # Duration score golden_dur = golden.end_time - golden.start_time cand_dur = cand.end_time - cand.start_time dur_score = 1.0 - abs(golden_dur - cand_dur) / max(golden_dur, 1) dur_score = max(0, min(1, dur_score / self.tolerance_duration)) # Attr score (simple Jaccard-like) golden_attrs = set(golden.attributes.keys()) cand_attrs = set(cand.attributes.keys()) attr_intersect = len(golden_attrs & cand_attrs) attr_union = len(golden_attrs | cand_attrs) attr_score = attr_intersect / max(attr_union, 1) attr_score = max(0, min(1, attr_score / self.tolerance_attrs)) # Kind & status exact kind_score = 1.0 if golden.kind == cand.kind else 0.0 status_score = 1.0 if golden.status == cand.status else 0.0 return (dur_score + attr_score + kind_score + status_score) / 4 class SyntheticMonitor: """Core monitoring engine.""" def __init__(self): self.golden_traces: List[Trace] = [] self.causality_rules: List[CausalityRule] = [] self.matchers: List[GoldenTraceMatcher] = [] def add_golden_trace(self, trace: Trace): """Register a golden trace.""" self.golden_traces.append(trace) self.matchers.append(GoldenTraceMatcher(trace)) def add_causality_rule(self, rule: CausalityRule): """Register a causality rule.""" self.causality_rules.append(rule) def monitor(self, trace: Trace) -> Dict[str, Any]: """Run full checks: causality + golden match. Returns report.""" report = { "trace_id": trace.trace_id, "timestamp": datetime.now().isoformat(), "causality_ok": True, "causality_violations": [], "golden_matches": [], "overall_score": 1.0, "alert": False } # Causality checks for rule in self.causality_rules: if not rule.check(trace): report["causality_ok"] = False report["causality_violations"].append(rule.name) # Golden trace matching best_score = 0.0 for matcher in self.matchers: score, devs = matcher.score_match(trace) report["golden_matches"].append({"golden_id": matcher.golden.trace_id, "score": score, "deviations": devs}) best_score = max(best_score, score) report["overall_score"] = best_score report["alert"] = not report["causality_ok"] or best_score < 0.8 return report # ============================================================================= # EXAMPLE: Synthetic E-Commerce Checkout Flow # ============================================================================= def generate_timestamp(base: datetime) -> int: return int(base.timestamp() * 1_000_000_000) def create_sample_trace(trace_id: str) -> Trace: base_time = datetime.now() spans = [ Span( trace_id=trace_id, span_id="span_auth", parent_span_id=None, name="auth_check", kind=SpanKind.SERVER, start_time=generate_timestamp(base_time), end_time=generate_timestamp(base_time + timedelta(milliseconds=50)), attributes={"user.id": "123", "method": "POST"}, events=[{"name": "user_authenticated", "time": generate_timestamp(base_time + timedelta(milliseconds=20))}], status={"code": "OK"}, links=[] ), Span( trace_id=trace_id, span_id="span_inv", parent_span_id="span_auth", name="inventory_check", kind=SpanKind.CLIENT, start_time=generate_timestamp(base_time + timedelta(milliseconds=60)), end_time=generate_timestamp(base_time + timedelta(milliseconds=120)), attributes={"product.id": "abc", "quantity": 1}, events=[{"name": "stock_reserved", "time": generate_timestamp(base_time + timedelta(milliseconds=100))}], status={"code": "OK"}, links=[] ), Span( trace_id=trace_id, span_id="span_pay", parent_span_id="span_auth", name="process_payment", kind=SpanKind.SERVER, start_time=generate_timestamp(base_time + timedelta(milliseconds=130)), end_time=generate_timestamp(base_time + timedelta(milliseconds=200)), attributes={"payment.id": "pay_456", "amount": 29.99}, events=[{"name": "charged", "time": generate_timestamp(base_time + timedelta(milliseconds=180))}], status={"code": "OK"}, links=[] ), Span( trace_id=trace_id, span_id="span_ship", parent_span_id=None, name="schedule_shipping", kind=SpanKind.PRODUCER, start_time=generate_timestamp(base_time + timedelta(milliseconds=210)), end_time=generate_timestamp(base_time + timedelta(milliseconds=250)), attributes={"order.id": "ord_789"}, events=[{"name": "shipped", "time": generate_timestamp(base_time + timedelta(milliseconds=240))}], status={"code": "OK"}, links=[{"trace_id": trace_id, "span_id": "span_pay"}] # Causal link ) ] return Trace(trace_id=trace_id, spans=spans, resource={"service.name": "checkout", "version": "1.0"}) # Pre-defined golden traces (happy paths) GOLDEN_TRACE_ID = str(uuid.uuid4()) GOLDEN_TRACE = create_sample_trace(GOLDEN_TRACE_ID) # Causality rules examples def rule_auth_before_db(trace: Trace): auth_span = next((s for s in trace.spans if s.name == "auth_check"), None) db_spans = [s for s in trace.spans if "db" in s.name.lower()] if auth_span and db_spans: if auth_span.end_time > min(s.start_time for s in db_spans): raise CausalityViolation("auth must precede DB ops") elif db_spans: raise CausalityViolation("DB ops without auth") def rule_payment_causes_shipping(trace: Trace): pay_span = next((s for s in trace.spans if s.name == "process_payment"), None) ship_span = next((s for s in trace.spans if s.name == "schedule_shipping"), None) if pay_span and ship_span and pay_span.end_time > ship_span.start_time: raise CausalityViolation("shipping cannot precede payment") RULES = [ CausalityRule("auth-before-db", rule_auth_before_db), CausalityRule("payment-before-shipping", rule_payment_causes_shipping), ] # ============================================================================= # USAGE DEMO # ============================================================================= def demo(): monitor = SyntheticMonitor() # Load golden traces & rules monitor.add_golden_trace(GOLDEN_TRACE) for rule in RULES: monitor.add_causality_rule(rule) # Test: Perfect match (golden trace itself) report_perfect = monitor.monitor(GOLDEN_TRACE) print("=== PERFECT GOLDEN TRACE ===") print(json.dumps(report_perfect, indent=2)) # Test: Anomalous trace (causality violation + deviation) bad_trace_id = str(uuid.uuid4()) bad_spans = GOLDEN_TRACE.spans.copy() # Swap times to violate causality bad_spans[3].start_time = generate_timestamp(datetime.now() - timedelta(milliseconds=100)) # Shipping too early bad_spans[3].end_time = generate_timestamp(datetime.now() - timedelta(milliseconds=50)) # Add extra attr deviation bad_spans[0].attributes["extra_malicious"] = "flag" bad_trace = Trace(trace_id=bad_trace_id, spans=bad_spans, resource=GOLDEN_TRACE.resource) report_bad = monitor.monitor(bad_trace) print("\n=== ANOMALOUS TRACE ===") print(json.dumps(report_bad, indent=2)) # Persist golden traces (e.g., to file) with open("golden_traces.json", "w") as f: json.dump([asdict(t) for t in monitor.golden_traces], f, default=str) print("\nGolden traces saved to 'golden_traces.json'") if __name__ == "__main__": demo()Quick Start Guide
-
Run the demo:
python this_script.py- Outputs reports for perfect vs. anomalous traces.
- Alerts on score < 0.8 or causality fails.
-
Extend Golden Traces:
new_golden = create_sample_trace(str(uuid.uuid4())) # Customize spans monitor.add_golden_trace(new_golden) -
Add Rules:
def my_rule(trace): ... # Raise CausalityViolation on fail monitor.add_causality_rule(CausalityRule("my-rule", my_rule)) -
Production Integration:
- Hook
monitor.monitor(trace)to OTLP receiver (e.g.,opentelemetry-sdk). - Export reports to Kafka/Prometheus.
- Generate more goldens synthetically via
create_sample_trace()variants.
- Hook
-
Customization:
- Tweak tolerances in
GoldenTraceMatcher. - Add ML-based matching (e.g., via
sentence-transformersfor span embeddings).
- Tweak tolerances in
This suite detects deviations like reordered spans (causality breaks), missing events, or perf regressions. Sherlock approves: the evidence never lies! 🔍
-
Other models, same or similar prompt
Responses were produced in different, uncontrolled contexts (system prompts, settings, prior turns). This is not a benchmark.