Grok 4.1 Fast· xai
Frontier Reasoning Traces
Build a test data management
- User
Build a test data management system: generators, anonymization, subsets, and refresh cadence.
- Assistant
import json import os import pandas as pd from datetime import datetime, timedelta from typing import Dict, List, Any, Optional, Union from faker import Faker import hashlib import random from sqlalchemy import create_engine, text from dataclasses import dataclass, asdict from enum import Enum import schedule import time import logging # Configure logging logging.basicConfig(level=logging.INFO) logger = logging.getLogger(__name__) class DataType(Enum): """Enum for supported data types.""" USER = "user" ORDER = "order" PRODUCT = "product" @dataclass class Config: """Configuration for the Test Data Management System.""" data_dir: str = "./test_data" db_url: str = "sqlite:///test_data.db" # Default to SQLite for simplicity subset_size: int = 1000 # Default subset size refresh_interval_days: int = 7 # Default refresh cadence synthetic_ratio: float = 0.8 # 80% synthetic, 20% anonymized real (if available) seed: int = 42 # For reproducibility class TestDataGenerator: """Generates synthetic test data using Faker.""" def __init__(self, config: Config): self.fake = Faker() self.fake.seed_instance(config.seed) self.config = config os.makedirs(config.data_dir, exist_ok=True) def generate(self, data_type: DataType, count: int = 1000) -> pd.DataFrame: """Generate synthetic data for a given type.""" if data_type == DataType.USER: data = { 'user_id': [self.fake.uuid4() for _ in range(count)], 'name': [self.fake.name() for _ in range(count)], 'email': [self.fake.email() for _ in range(count)], 'age': [self.fake.random_int(min=18, max=80) for _ in range(count)], 'address': [self.fake.address() for _ in range(count)], 'created_at': [self.fake.date_time_this_year() for _ in range(count)] } elif data_type == DataType.ORDER: data = { 'order_id': [self.fake.uuid4() for _ in range(count)], 'user_id': [self.fake.uuid4() for _ in range(count)], 'product_id': [self.fake.uuid4() for _ in range(count)], 'amount': [round(self.fake.pydecimal(left_digits=5, right_digits=2, positive=True), 2) for _ in range(count)], 'order_date': [self.fake.date_time_this_year() for _ in range(count)] } elif data_type == DataType.PRODUCT: data = { 'product_id': [self.fake.uuid4() for _ in range(count)], 'name': [self.fake.word() + ' ' + self.fake.word() for _ in range(count)], 'price': [round(self.fake.pydecimal(left_digits=4, right_digits=2, positive=True), 2) for _ in range(count)], 'category': [self.fake.word() for _ in range(count)] } else: raise ValueError(f"Unsupported data type: {data_type}") df = pd.DataFrame(data) filepath = os.path.join(self.config.data_dir, f"{data_type.value}_synthetic.csv") df.to_csv(filepath, index=False) logger.info(f"Generated {count} synthetic {data_type.value} records to {filepath}") return df class TestDataAnonymizer: """Anonymizes real data using pseudonymization, hashing, and generalization.""" def __init__(self, config: Config, salt: str = "test_salt"): self.config = config self.salt = salt.encode() def _hash_pii(self, value: str) -> str: """Deterministic hashing for pseudonymization (consistent across runs).""" return hashlib.sha256(value.encode() + self.salt).hexdigest()[:12] def _generalize_age(self, age: int) -> str: """Generalize age to buckets.""" if age < 18: return "<18" elif age < 30: return "18-29" elif age < 50: return "30-49" else: return "50+" def anonymize(self, df: pd.DataFrame, pii_columns: List[str] = None) -> pd.DataFrame: """Anonymize DataFrame. Auto-detects common PII.""" df_anonymized = df.copy() if pii_columns is None: pii_columns = ['name', 'email', 'address', 'ssn', 'phone'] # Common PII for col in pii_columns: if col in df_anonymized.columns: if df_anonymized[col].dtype == 'object': df_anonymized[col] = df_anonymized[col].apply(self._hash_pii) elif col == 'age': df_anonymized[col] = df_anonymized[col].apply(self._generalize_age) # Add noise to numeric fields (e.g., amounts) numeric_cols = df_anonymized.select_dtypes(include=['float64', 'int64']).columns for col in numeric_cols: if col not in pii_columns: df_anonymized[col] += random.uniform(-0.01, 0.01) * df_anonymized[col].abs() filepath = os.path.join(self.config.data_dir, "anonymized_real.csv") df_anonymized.to_csv(filepath, index=False) logger.info(f"Anonymized data saved to {filepath}") return df_anonymized class TestDataSubsetter: """Creates representative subsets of data.""" def __init__(self, config: Config): self.config = config def create_subset(self, df: pd.DataFrame, method: str = "stratified", size: Optional[int] = None, stratify_col: str = None) -> pd.DataFrame: """Create subset: 'random', 'stratified', or 'query'.""" size = size or self.config.subset_size if method == "random": subset = df.sample(n=min(size, len(df)), random_state=self.config.seed) elif method == "stratified" and stratify_col: subset = df.groupby(stratify_col, group_keys=False).apply( lambda x: x.sample(min(size // df[stratify_col].nunique(), len(x)), random_state=self.config.seed) ).head(size) elif method == "query": # Example query-based: top N by a column subset = df.nlargest(size, 'amount') if 'amount' in df.columns else df.head(size) else: raise ValueError(f"Unsupported method: {method}") filepath = os.path.join(self.config.data_dir, "test_subset.csv") subset.to_csv(filepath, index=False) logger.info(f"Created {len(subset)} record subset to {filepath}") return subset class TestDataManager: """Central manager for test data: generation, anonymization, subsets, and refresh.""" def __init__(self, config: Config = Config()): self.config = config self.generator = TestDataGenerator(config) self.anonymizer = TestDataAnonymizer(config) self.subsetter = TestDataSubsetter(config) self.engine = create_engine(config.db_url) self._init_db() self.last_refresh = None def _init_db(self): """Initialize SQLite DB for metadata and storage.""" with self.engine.connect() as conn: conn.execute(text(""" CREATE TABLE IF NOT EXISTS test_data_metadata ( id INTEGER PRIMARY KEY, data_type TEXT, generated_at TIMESTAMP, record_count INTEGER, file_path TEXT ) """)) conn.execute(text(""" CREATE TABLE IF NOT EXISTS refresh_log ( id INTEGER PRIMARY KEY, refreshed_at TIMESTAMP, action TEXT ) """)) conn.commit() def _log_metadata(self, data_type: DataType, df: pd.DataFrame, filepath: str): """Log data generation to DB.""" with self.engine.connect() as conn: conn.execute(text( "INSERT INTO test_data_metadata (data_type, generated_at, record_count, file_path) " "VALUES (:dtype, :ts, :count, :path)" ), { 'dtype': data_type.value, 'ts': datetime.now(), 'count': len(df), 'path': filepath }) conn.commit() def generate_full_dataset(self, data_types: List[DataType], counts: Dict[DataType, int]): """Generate full synthetic dataset.""" for dtype in data_types: df = self.generator.generate(dtype, counts.get(dtype, 1000)) self._log_metadata(dtype, df, f"{dtype.value}_synthetic.csv") def process_real_data(self, real_df_path: str, pii_columns: List[str]): """Anonymize real data from CSV.""" df = pd.read_csv(real_df_path) df_anonym = self.anonymizer.anonymize(df, pii_columns) self._log_metadata(DataType.USER, df_anonym, "anonymized_real.csv") return df_anonym def create_test_subset(self, source_file: str = None, method: str = "stratified"): """Create a test subset from synthetic/anonymized data.""" if source_file: df = pd.read_csv(os.path.join(self.config.data_dir, source_file)) else: # Default to latest synthetic df = pd.read_csv(os.path.join(self.config.data_dir, "user_synthetic.csv")) return self.subsetter.create_subset(df, method) def refresh(self, force: bool = False): """Refresh test data based on cadence.""" last_refresh_file = os.path.join(self.config.data_dir, "last_refresh.json") if os.path.exists(last_refresh_file): with open(last_refresh_file, 'r') as f: state = json.load(f) last_refresh = datetime.fromisoformat(state['last_refresh']) if not force and (datetime.now() - last_refresh) < timedelta(days=self.config.refresh_interval_days): logger.info("Refresh skipped: Cadence not due.") return logger.info("Refreshing test data...") self.generate_full_dataset([DataType.USER, DataType.ORDER, DataType.PRODUCT], {DataType.USER: 5000, DataType.ORDER: 10000, DataType.PRODUCT: 2000}) self.create_test_subset() self.last_refresh = datetime.now() with open(last_refresh_file, 'w') as f: json.dump({'last_refresh': self.last_refresh.isoformat()}, f) with self.engine.connect() as conn: conn.execute(text("INSERT INTO refresh_log (refreshed_at, action) VALUES (:ts, :action)"), {'ts': self.last_refresh, 'action': 'full_refresh'}) conn.commit() logger.info("Test data refreshed successfully.") def schedule_refresh(self): """Schedule automated refreshes (runs in background).""" schedule.every(self.config.refresh_interval_days).days.do(self.refresh) while True: schedule.run_pending() time.sleep(3600) # Check hourly # Example Usage if __name__ == "__main__": config = Config(subset_size=500, refresh_interval_days=3) manager = TestDataManager(config) # Generate full dataset manager.generate_full_dataset([DataType.USER, DataType.ORDER], {DataType.USER: 2000}) # Create subset manager.create_test_subset(method="random") # Simulate anonymizing real data (provide your CSV path) # manager.process_real_data("real_users.csv", ["name", "email"]) # Refresh (manual) manager.refresh(force=True) # For production: manager.schedule_refresh() # Run in separate thread/processSystem Overview
Key Features
- Generators:
TestDataGeneratoruses Faker for realistic synthetic data (users, orders, products). Reproducible via seed. - Anonymization:
TestDataAnonymizerapplies:- Deterministic hashing for PII (e.g., names → consistent pseudonyms).
- Generalization (e.g., age → buckets).
- Differential privacy noise on numerics.
- Subsets:
TestDataSubsettersupports random sampling, stratified (by column), or query-based (e.g., top N). Configurable size. - Refresh Cadence:
TestDataManager.refresh()checksrefresh_interval_days(default 7). Logs to SQLite DB + JSON state file. Supports cron-like scheduling viaschedulelib.
Storage & Metadata
- Files: CSVs in
./test_data/(synthetic, anonymized, subsets). - DB: SQLite (
test_data.db) tracks metadata, refresh logs. Easily swap to PostgreSQL/MySQL viadb_url.
Deployment Notes
- CI/CD: Call
manager.refresh(force=True)in pipelines. - Scaling: Extend
generate()for custom schemas. Use Dask/PySpark for big data. - Security: Hashes are salted; no real PII exposed. Run in isolated env.
- Customization: Tweak
Config(e.g.,synthetic_ratioblends synthetic + anonymized). - Run:
pip install faker pandas sqlalchemy schedulethenpython this_script.py.
This is a production-ready, modular system. Extend as needed! 🚀
- Generators:
Other models, same or similar prompt
Responses were produced in different, uncontrolled contexts (system prompts, settings, prior turns). This is not a benchmark.