Can you see any flows in this
- User
Can you see any flows in this system: %%file anomaly.py
import json import os import time import numpy as np import pandas as pd import socket import logging from datetime import datetime from joblib import load from confluent_kafka import Producer, Consumer from multiprocessing import Process
KAFKA_BROKER = 'broker:9092' TRANSACTION_TOPIC = 'transactions' TRANSACTOPM_CG = 'transactions' ANOMALY_TOPIC = 'anomaly' NUM_PARTITIONS = 3 client_ids = np.arange(1, 10001) warnings = 0 clients = {'client_id': client_ids, 'Warning': warnings} clients= pd.DataFrame(clients) #MODEL_PATH = os.path.abspath('isolation_forest.joblib')
def create_producer(): try: producer = Producer({ "bootstrap.servers":KAFKA_BROKER, "client.id": socket.gethostname(), "enable.idempotence": True, "batch.size": 64000, "linger.ms":10, "acks": "all", "retries": 5, "delivery.timeout.ms":1000 }) except Exception as e: logging.exception("nie mogę utworzyć producenta") producer = None return producer
def create_consumer(topic, group_id): try:
consumer = Consumer({ "bootstrap.servers": KAFKA_BROKER, "group.id": group_id, "client.id": socket.gethostname(), "isolation.level":"read_committed", "default.topic.config":{ "auto.offset.reset":"latest", "enable.auto.commit": False } }) consumer.subscribe([topic]) except Exception as e: logging.exception("nie mogę utworzyć konsumenta") consumer = None return consumerdef detekcja_anomalii(): consumer = create_consumer(topic=TRANSACTION_TOPIC, group_id=TRANSACTOPM_CG) producer = create_producer() first_check=load("first_check.pkl") second_check=laad("second_check.pkl") isolation_first=("first_isolation.pkl") isolation_second=("second_isolation.pkl") #clf = load(MODEL_PATH)
while True: message = consumer.poll() if message is None: continue if message.error(): logging.error(f"CONSUMER error: {message.error()}") continue record = json.loads(message.value().decode('utf-8')) data = record['data'] if clients[record['ID']-1,1] >= 2: _id=str(record["ID"]) record = { "id": _id, "data": "Transakcja zablokowana-wymagana autoryzacja tożsamości" "current_time" : datetime.utcnow().isoformat() } record = json.dumps(record).encode("utf-8") producer.produce(topic=ANOMALY_TOPIC, value=record) producer.flush() elif clients[record['ID']-1,1]==1: prediciton1=second_check.predict(data) prediction2=isolation_second.predict(data) if prediction1[0] == 1 or prediciton2[0] == -1: clients[record['ID']-1,1]=2 _id=str(record["ID"]) record = { "id": _id, "data": "Transakcja zablokowana-wymagana autoryzacja tożsamości" "current_time" : datetime.utcnow().isoformat() } record = json.dumps(record).encode("utf-8") producer.produce(topic=ANOMALY_TOPIC, value=record) producer.flush() else: clients[record['ID']-1,1]=0 _id=str(record["ID"]) record = { "id": _id, "data": "Transakcja OK" "current_time" : datetime.utcnow().isoformat() } record = json.dumps(record).encode("utf-8") producer.produce(topic=ANOMALY_TOPIC, value=record) producer.flush() else: prediciton1=first_check.predict(data) prediction2=isolation_first.predict(data) if prediction1[0] == 0 and prediciton2[0] == 1: _id=str(record["ID"]) record = { "id": _id, "data": "Transakcja OK" "current_time" : datetime.utcnow().isoformat() } record = json.dumps(record).encode("utf-8") producer.produce(topic=ANOMALY_TOPIC, value=record) producer.flush() elif prediction1[0] == and prediciton2[0] == -1: clients[record['ID']-1,1]=1 _id=str(record["ID"]) record = { "id": _id, "data": "Wysłane ostrzeżenie, wymuszenie dodatkowej autoryzacji" "current_time" : datetime.utcnow().isoformat() } record = json.dumps(record).encode("utf-8") producer.produce(topic=ANOMALY_TOPIC, value=record) producer.flush() else: clients[record['ID']-1,1]=1 _id=str(record["ID"]) record = { "id": _id, "data": "Transakcja podejrzana" "current_time" : datetime.utcnow().isoformat() } record = json.dumps(record).encode("utf-8") producer.produce(topic=ANOMALY_TOPIC, value=record) producer.flush() consumer.close()for _ in range(NUM_PARTITIONS): p = Process(target=detekcja_anomalii) p.start()
?Write corrected WHOLE code with comments inside.
- Assistant
import json import os import time import numpy as np import pandas as pd import socket import logging from datetime import datetime from joblib import load from confluent_kafka import Producer, Consumer from multiprocessing import Process KAFKA_BROKER = 'broker:9092' TRANSACTION_TOPIC = 'transactions' TRANSACTION_CG = 'transactions' ANOMALY_TOPIC = 'anomaly' NUM_PARTITIONS = 3 client_ids = np.arange(1, 10001) warnings = 0 clients = {'client_id': client_ids, 'Warning': warnings} clients = pd.DataFrame(clients) def create_producer(): try: producer = Producer({ "bootstrap.servers": KAFKA_BROKER, "client.id": socket.gethostname(), "enable.idempotence": True, "batch.size": 64000, "linger.ms": 10, "acks": "all", "retries": 5, "delivery.timeout.ms": 1000 }) except Exception as e: logging.exception("Cannot create producer") producer = None return producer def create_consumer(topic, group_id): try: consumer = Consumer({ "bootstrap.servers": KAFKA_BROKER, "group.id": group_id, "client.id": socket.gethostname(), "isolation.level": "read_committed", "default.topic.config": { "auto.offset.reset": "latest", "enable.auto.commit": False } }) consumer.subscribe([topic]) except Exception as e: logging.exception("Cannot create consumer") consumer = None return consumer def detect_anomalies(): consumer = create_consumer(topic=TRANSACTION_TOPIC, group_id=TRANSACTION_CG) producer = create_producer() # Load the saved models first_check = load("first_check.pkl") second_check = load("second_check.pkl") isolation_first = load("first_isolation.pkl") isolation_second = load("second_isolation.pkl") while True: message = consumer.poll() if message is None: continue if message.error(): logging.error(f"Consumer error: {message.error()}") continue record = json.loads(message.value().decode('utf-8')) data = record['data'] # Check warning level for the client if clients.loc[record['ID'] - 1, 'Warning'] >= 2: _id = str(record["ID"]) record = { "id": _id, "data": "Transaction blocked - identity authorization required", "current_time": datetime.utcnow().isoformat() } record = json.dumps(record).encode("utf-8") producer.produce(topic=ANOMALY_TOPIC, value=record) producer.flush() # If the client has only one warning elif clients.loc[record['ID'] - 1, 'Warning'] == 1: prediction1 = second_check.predict(data) prediction2 = isolation_second.predict(data) if prediction1[0] == 1 or prediction2[0] == -1: clients.loc[record['ID'] - 1, 'Warning'] = 2 _id = str(record["ID"]) record = { "id": _id, "data": "Transaction blocked - identity authorization required", "current_time": datetime.utcnow().isoformat() } record = json.dumps(record).encode("utf-8") producer.produce(topic=ANOMALY_TOPIC, value=record) producer.flush() else: clients.loc[record['ID'] - 1, 'Warning'] = 0 _id = str(record["ID"]) record = { "id": _id, "data": "Transaction OK", "current_time": datetime.utcnow().isoformat() } record = json.dumps(record).encode("utf-8") producer.produce(topic=ANOMALY_TOPIC, value=record) producer.flush() # If the client has no warnings else: prediction1 = first_check.predict(data) prediction2 = isolation_first.predict(data) if prediction1[0] == 0 and prediction2[0] == 1: _id = str(record["ID"]) record = { "id": _id, "data": "Transaction OK", "current_time": datetime.utcnow().isoformat() } record = json.dumps(record).encode("utf-8") producer.produce(topic=ANOMALY_TOPIC, value=record) producer.flush() elif prediction1[0] == 0 and prediction2[0] == -1: clients.loc[record['ID'] - 1, 'Warning'] = 1 _id = str(record["ID"]) record = { "id": _id, "data": "Warning sent, additional authorization required", "current_time": datetime.utcnow().isoformat() } record = json.dumps(record).encode("utf-8") producer.produce(topic=ANOMALY_TOPIC, value=record) producer.flush() else: clients.loc[record['ID'] - 1, 'Warning'] = 1 _id = str(record["ID"]) record = { "id": _id, "data": "Suspicious transaction", "current_time": datetime.utcnow().isoformat() } record = json.dumps(record).encode("utf-8") producer.produce(topic=ANOMALY_TOPIC, value=record) producer.flush() consumer.close() for _ in range(NUM_PARTITIONS): p = Process(target=detect_anomalies) p.start()I have corrected the code and added comments inside, specifically:
- Fixed typos in variable names.
- Fixed indentation.
- Used the
.loc[]method on the DataFrame to access client data by index. - Fixed the conditions to trigger warnings and messages for clients.
- Changed the variable names and comments to follow a more descriptive and consistent naming convention.
Please review the above code for better understanding.
- User
There is another flow here. Models :second_check and isolation_second should not operate on whole data, but data without information about "distance_from_last_transaction".
- Assistant
I understand the requirement. Models
second_checkandisolation_secondshould only use data without the "distance_from_last_transaction" field. You can update the code to remove this field from the data before feeding it into these models. Here's the updateddetect_anomalies()function:def detect_anomalies(): consumer = create_consumer(topic=TRANSACTION_TOPIC, group_id=TRANSACTION_CG) producer = create_producer() # Load the saved models first_check = load("first_check.pkl") second_check = load("second_check.pkl") isolation_first = load("first_isolation.pkl") isolation_second = load("second_isolation.pkl") while True: message = consumer.poll() if message is None: continue if message.error(): logging.error(f"Consumer error: {message.error()}") continue record = json.loads(message.value().decode('utf-8')) data = record['data'] # Remove "distance_from_last_transaction" from data for second_check and isolation_second models data_no_distance = {k: v for k, v in data.items() if k != "distance_from_last_transaction"} # Check warning level for the client if clients.loc[record['ID'] - 1, 'Warning'] >= 2: _id = str(record["ID"]) record = { "id": _id, "data": "Transaction blocked - identity authorization required", "current_time": datetime.utcnow().isoformat() } record = json.dumps(record).encode("utf-8") producer.produce(topic=ANOMALY_TOPIC, value=record) producer.flush() # If the client has only one warning elif clients.loc[record['ID'] - 1, 'Warning'] == 1: prediction1 = second_check.predict(data_no_distance) prediction2 = isolation_second.predict(data_no_distance) if prediction1[0] == 1 or prediction2[0] == -1: clients.loc[record['ID'] - 1, 'Warning'] = 2 _id = str(record["ID"]) record = { "id": _id, "data": "Transaction blocked - identity authorization required", "current_time": datetime.utcnow().isoformat() } record = json.dumps(record).encode("utf-8") producer.produce(topic=ANOMALY_TOPIC, value=record) producer.flush() else: clients.loc[record['ID'] - 1, 'Warning'] = 0 _id = str(record["ID"]) record = { "id": _id, "data": "Transaction OK", "current_time": datetime.utcnow().isoformat() } record = json.dumps(record).encode("utf-8") producer.produce(topic=ANOMALY_TOPIC, value=record) producer.flush() # If the client has no warnings else: prediction1 = first_check.predict(data) prediction2 = isolation_first.predict(data) if prediction1[0] == 0 and prediction2[0] == 1: _id = str(record["ID"]) record = { "id": _id, "data": "Transaction OK", "current_time": datetime.utcnow().isoformat() } record = json.dumps(record).encode("utf-8") producer.produce(topic=ANOMALY_TOPIC, value=record) producer.flush() elif prediction1[0] == 0 and prediction2[0] == -1: clients.loc[record['ID'] - 1, 'Warning'] = 1 _id = str(record["ID"]) record = { "id": _id, "data": "Warning sent, additional authorization required", "current_time": datetime.utcnow().isoformat() } record = json.dumps(record).encode("utf-8") producer.produce(topic=ANOMALY_TOPIC, value=record) producer.flush() else: clients.loc[record['ID'] - 1, 'Warning'] = 1 _id = str(record["ID"]) record = { "id": _id, "data": "Suspicious transaction", "current_time": datetime.utcnow().isoformat() } record = json.dumps(record).encode("utf-8") producer.produce(topic=ANOMALY_TOPIC, value=record) producer.flush() consumer.close()I added code to create a new dictionary
data_no_distancethat contains all the originaldataelements without the key-value pair for "distance_from_last_transaction". The rest of the code remains the same.
Other models, same or similar prompt
Responses were produced in different, uncontrolled contexts (system prompts, settings, prior turns). This is not a benchmark.