Skip to content
TheChatArchiveThe Chat Archive
Grok 4.1 Fast· xaiHugging Face Frontier Reasoning Traces

Build a synthetic monitoring suite: golden traces and causality checks.

  1. User

    Build a synthetic monitoring suite: golden traces and causality checks.

  2. Assistant
    #!/usr/bin/env python3
    """
    Synthetic Monitoring Suite: Golden Traces & Causality Checks
    ===========================================================
    
    This suite provides a framework for synthetic monitoring in distributed systems.
    It uses OpenTelemetry-inspired trace models (spans with attributes, events, links).
    
    Key Components:
    - **Golden Traces**: Pre-defined exemplary traces representing "happy path" behaviors.
      These are synthetic baselines for anomaly detection.
    - **Causality Checks**: Rule-based validation of causal relationships (e.g., span ordering,
      parent-child hierarchies, duration SLAs, attribute propagation, event sequencing).
    - **Monitoring Engine**: Compares runtime traces against golden traces, scoring deviations
      and flagging causality violations.
    
    Usage:
    1. Define golden traces for your services.
    2. Ingest runtime traces (e.g., from OpenTelemetry exporter).
    3. Run checks and get alerts/reports.
    
    Extensible for production: Integrate with Prometheus, Grafana, or Kafka for real-time monitoring.
    """
    
    import json
    import uuid
    from dataclasses import dataclass, asdict
    from typing import List, Dict, Any, Optional, Tuple
    from enum import Enum
    from datetime import datetime, timedelta
    import hashlib
    
    
    class SpanKind(Enum):
        INTERNAL = "INTERNAL"
        SERVER = "SERVER"
        CLIENT = "CLIENT"
        PRODUCER = "PRODUCER"
        CONSUMER = "CONSUMER"
    
    
    @dataclass
    class Span:
        trace_id: str
        span_id: str
        parent_span_id: Optional[str]
        name: str
        kind: SpanKind
        start_time: int  # Unix ns
        end_time: int    # Unix ns
        attributes: Dict[str, Any]
        events: List[Dict[str, Any]]  # timestamped events
        status: Dict[str, Any]  # e.g., {"code": "OK"}
        links: List[Dict[str, Any]]
    
    
    @dataclass
    class Trace:
        trace_id: str
        spans: List[Span]
        resource: Dict[str, Any]  # e.g., service name, version
    
    
    class CausalityViolation(Exception):
        pass
    
    
    class CausalityRule:
        """Defines a causal invariant (e.g., 'auth span must precede db query')."""
        def __init__(self, name: str, condition: callable):
            self.name = name
            self.condition = condition  # fn(trace: Trace) -> bool or raises CausalityViolation
    
        def check(self, trace: Trace) -> bool:
            try:
                self.condition(trace)
                return True
            except CausalityViolation:
                return False
    
    
    class GoldenTraceMatcher:
        """Fuzzy matcher for golden traces with tolerance for minor variations."""
        
        def __init__(self, golden_trace: Trace, tolerance_attrs: float = 0.8, tolerance_duration: float = 0.2):
            self.golden = golden_trace
            self.tolerance_attrs = tolerance_attrs  # 80% attr match
            self.tolerance_duration = tolerance_duration  # ±20% duration
        
        def score_match(self, candidate: Trace) -> Tuple[float, List[str]]:
            """Returns match score (0-1) and deviation reasons."""
            if candidate.trace_id != self.golden.trace_id:
                return 0.0, ["trace_id mismatch"]
            
            golden_spans_by_name = {s.name: s for s in self.golden.spans}
            candidate_spans_by_name = {s.name: s for s in candidate.spans}
            
            deviations = []
            total_score = 0.0
            span_count = 0
            
            for name, golden_span in golden_spans_by_name.items():
                if name not in candidate_spans_by_name:
                    deviations.append(f"missing span: {name}")
                    continue
                
                cand_span = candidate_spans_by_name[name]
                span_score = self._score_span(golden_span, cand_span)
                total_score += span_score
                span_count += 1
                if span_score < 0.7:
                    deviations.append(f"low score on {name}: {span_score}")
            
            avg_score = total_score / max(span_count, 1)
            return avg_score, deviations
        
        def _score_span(self, golden: Span, cand: Span) -> float:
            # Duration score
            golden_dur = golden.end_time - golden.start_time
            cand_dur = cand.end_time - cand.start_time
            dur_score = 1.0 - abs(golden_dur - cand_dur) / max(golden_dur, 1)
            dur_score = max(0, min(1, dur_score / self.tolerance_duration))
            
            # Attr score (simple Jaccard-like)
            golden_attrs = set(golden.attributes.keys())
            cand_attrs = set(cand.attributes.keys())
            attr_intersect = len(golden_attrs & cand_attrs)
            attr_union = len(golden_attrs | cand_attrs)
            attr_score = attr_intersect / max(attr_union, 1)
            attr_score = max(0, min(1, attr_score / self.tolerance_attrs))
            
            # Kind & status exact
            kind_score = 1.0 if golden.kind == cand.kind else 0.0
            status_score = 1.0 if golden.status == cand.status else 0.0
            
            return (dur_score + attr_score + kind_score + status_score) / 4
    
    
    class SyntheticMonitor:
        """Core monitoring engine."""
        
        def __init__(self):
            self.golden_traces: List[Trace] = []
            self.causality_rules: List[CausalityRule] = []
            self.matchers: List[GoldenTraceMatcher] = []
        
        def add_golden_trace(self, trace: Trace):
            """Register a golden trace."""
            self.golden_traces.append(trace)
            self.matchers.append(GoldenTraceMatcher(trace))
        
        def add_causality_rule(self, rule: CausalityRule):
            """Register a causality rule."""
            self.causality_rules.append(rule)
        
        def monitor(self, trace: Trace) -> Dict[str, Any]:
            """Run full checks: causality + golden match. Returns report."""
            report = {
                "trace_id": trace.trace_id,
                "timestamp": datetime.now().isoformat(),
                "causality_ok": True,
                "causality_violations": [],
                "golden_matches": [],
                "overall_score": 1.0,
                "alert": False
            }
            
            # Causality checks
            for rule in self.causality_rules:
                if not rule.check(trace):
                    report["causality_ok"] = False
                    report["causality_violations"].append(rule.name)
            
            # Golden trace matching
            best_score = 0.0
            for matcher in self.matchers:
                score, devs = matcher.score_match(trace)
                report["golden_matches"].append({"golden_id": matcher.golden.trace_id, "score": score, "deviations": devs})
                best_score = max(best_score, score)
            
            report["overall_score"] = best_score
            report["alert"] = not report["causality_ok"] or best_score < 0.8
            
            return report
    
    
    # =============================================================================
    # EXAMPLE: Synthetic E-Commerce Checkout Flow
    # =============================================================================
    
    def generate_timestamp(base: datetime) -> int:
        return int(base.timestamp() * 1_000_000_000)
    
    def create_sample_trace(trace_id: str) -> Trace:
        base_time = datetime.now()
        spans = [
            Span(
                trace_id=trace_id,
                span_id="span_auth",
                parent_span_id=None,
                name="auth_check",
                kind=SpanKind.SERVER,
                start_time=generate_timestamp(base_time),
                end_time=generate_timestamp(base_time + timedelta(milliseconds=50)),
                attributes={"user.id": "123", "method": "POST"},
                events=[{"name": "user_authenticated", "time": generate_timestamp(base_time + timedelta(milliseconds=20))}],
                status={"code": "OK"},
                links=[]
            ),
            Span(
                trace_id=trace_id,
                span_id="span_inv",
                parent_span_id="span_auth",
                name="inventory_check",
                kind=SpanKind.CLIENT,
                start_time=generate_timestamp(base_time + timedelta(milliseconds=60)),
                end_time=generate_timestamp(base_time + timedelta(milliseconds=120)),
                attributes={"product.id": "abc", "quantity": 1},
                events=[{"name": "stock_reserved", "time": generate_timestamp(base_time + timedelta(milliseconds=100))}],
                status={"code": "OK"},
                links=[]
            ),
            Span(
                trace_id=trace_id,
                span_id="span_pay",
                parent_span_id="span_auth",
                name="process_payment",
                kind=SpanKind.SERVER,
                start_time=generate_timestamp(base_time + timedelta(milliseconds=130)),
                end_time=generate_timestamp(base_time + timedelta(milliseconds=200)),
                attributes={"payment.id": "pay_456", "amount": 29.99},
                events=[{"name": "charged", "time": generate_timestamp(base_time + timedelta(milliseconds=180))}],
                status={"code": "OK"},
                links=[]
            ),
            Span(
                trace_id=trace_id,
                span_id="span_ship",
                parent_span_id=None,
                name="schedule_shipping",
                kind=SpanKind.PRODUCER,
                start_time=generate_timestamp(base_time + timedelta(milliseconds=210)),
                end_time=generate_timestamp(base_time + timedelta(milliseconds=250)),
                attributes={"order.id": "ord_789"},
                events=[{"name": "shipped", "time": generate_timestamp(base_time + timedelta(milliseconds=240))}],
                status={"code": "OK"},
                links=[{"trace_id": trace_id, "span_id": "span_pay"}]  # Causal link
            )
        ]
        return Trace(trace_id=trace_id, spans=spans, resource={"service.name": "checkout", "version": "1.0"})
    
    
    # Pre-defined golden traces (happy paths)
    GOLDEN_TRACE_ID = str(uuid.uuid4())
    GOLDEN_TRACE = create_sample_trace(GOLDEN_TRACE_ID)
    
    # Causality rules examples
    def rule_auth_before_db(trace: Trace):
        auth_span = next((s for s in trace.spans if s.name == "auth_check"), None)
        db_spans = [s for s in trace.spans if "db" in s.name.lower()]
        if auth_span and db_spans:
            if auth_span.end_time > min(s.start_time for s in db_spans):
                raise CausalityViolation("auth must precede DB ops")
        elif db_spans:
            raise CausalityViolation("DB ops without auth")
    
    def rule_payment_causes_shipping(trace: Trace):
        pay_span = next((s for s in trace.spans if s.name == "process_payment"), None)
        ship_span = next((s for s in trace.spans if s.name == "schedule_shipping"), None)
        if pay_span and ship_span and pay_span.end_time > ship_span.start_time:
            raise CausalityViolation("shipping cannot precede payment")
    
    RULES = [
        CausalityRule("auth-before-db", rule_auth_before_db),
        CausalityRule("payment-before-shipping", rule_payment_causes_shipping),
    ]
    
    
    # =============================================================================
    # USAGE DEMO
    # =============================================================================
    
    def demo():
        monitor = SyntheticMonitor()
        
        # Load golden traces & rules
        monitor.add_golden_trace(GOLDEN_TRACE)
        for rule in RULES:
            monitor.add_causality_rule(rule)
        
        # Test: Perfect match (golden trace itself)
        report_perfect = monitor.monitor(GOLDEN_TRACE)
        print("=== PERFECT GOLDEN TRACE ===")
        print(json.dumps(report_perfect, indent=2))
        
        # Test: Anomalous trace (causality violation + deviation)
        bad_trace_id = str(uuid.uuid4())
        bad_spans = GOLDEN_TRACE.spans.copy()
        # Swap times to violate causality
        bad_spans[3].start_time = generate_timestamp(datetime.now() - timedelta(milliseconds=100))  # Shipping too early
        bad_spans[3].end_time = generate_timestamp(datetime.now() - timedelta(milliseconds=50))
        # Add extra attr deviation
        bad_spans[0].attributes["extra_malicious"] = "flag"
        bad_trace = Trace(trace_id=bad_trace_id, spans=bad_spans, resource=GOLDEN_TRACE.resource)
        
        report_bad = monitor.monitor(bad_trace)
        print("\n=== ANOMALOUS TRACE ===")
        print(json.dumps(report_bad, indent=2))
        
        # Persist golden traces (e.g., to file)
        with open("golden_traces.json", "w") as f:
            json.dump([asdict(t) for t in monitor.golden_traces], f, default=str)
        print("\nGolden traces saved to 'golden_traces.json'")
    
    
    if __name__ == "__main__":
        demo()
    

    Quick Start Guide

    1. Run the demo: python this_script.py

      • Outputs reports for perfect vs. anomalous traces.
      • Alerts on score < 0.8 or causality fails.
    2. Extend Golden Traces:

      new_golden = create_sample_trace(str(uuid.uuid4()))  # Customize spans
      monitor.add_golden_trace(new_golden)
      
    3. Add Rules:

      def my_rule(trace): ...  # Raise CausalityViolation on fail
      monitor.add_causality_rule(CausalityRule("my-rule", my_rule))
      
    4. Production Integration:

      • Hook monitor.monitor(trace) to OTLP receiver (e.g., opentelemetry-sdk).
      • Export reports to Kafka/Prometheus.
      • Generate more goldens synthetically via create_sample_trace() variants.
    5. Customization:

      • Tweak tolerances in GoldenTraceMatcher.
      • Add ML-based matching (e.g., via sentence-transformers for span embeddings).

    This suite detects deviations like reordered spans (causality breaks), missing events, or perf regressions. Sherlock approves: the evidence never lies! 🔍

Responses were produced in different, uncontrolled contexts (system prompts, settings, prior turns). This is not a benchmark.