Can you see any flaws in this
- User
Can you see any flaws in this system: %%file anomaly.py
import json import os import time import numpy as np import socket import logging from datetime import datetime from joblib import load from confluent_kafka import Producer, Consumer from multiprocessing import Process
KAFKA_BROKER = 'broker:9092' TRANSACTION_TOPIC = 'transactions' TRANSACTOPM_CG = 'transactions' ANOMALY_TOPIC = 'anomaly' NUM_PARTITIONS = 3 client_ids = np.arange(1, 10001) warnings = 0 clients = {'client_id': client_ids, 'Warning': warnings} clients= pd.DataFrame(clients) #MODEL_PATH = os.path.abspath('isolation_forest.joblib')
def create_producer(): try: producer = Producer({ "bootstrap.servers":KAFKA_BROKER, "client.id": socket.gethostname(), "enable.idempotence": True, "batch.size": 64000, "linger.ms":10, "acks": "all", "retries": 5, "delivery.timeout.ms":1000 }) except Exception as e: logging.exception("nie mogę utworzyć producenta") producer = None return producer
def create_consumer(topic, group_id): try:
consumer = Consumer({ "bootstrap.servers": KAFKA_BROKER, "group.id": group_id, "client.id": socket.gethostname(), "isolation.level":"read_committed", "default.topic.config":{ "auto.offset.reset":"latest", "enable.auto.commit": False } }) consumer.subscribe([topic]) except Exception as e: logging.exception("nie mogę utworzyć konsumenta") consumer = None return consumerdef detekcja_anomalii(): consumer = create_consumer(topic=TRANSACTION_TOPIC, group_id=TRANSACTOPM_CG) producer = create_producer() first_check=load("first_check.pkl") second_check=laad("second_check.pkl") isolation_first=("first_isolation.pkl") isolation_second=("second_isolation.pkl") #clf = load(MODEL_PATH)
while True: message = consumer.poll() if message is None: continue if message.error(): logging.error(f"CONSUMER error: {message.error()}") continue record = json.loads(message.value().decode('utf-8')) data = record['data'] if clients[record['ID']-1,1] >= 2: _id=str(record["ID"]) record = { "id": _id, "data": "Transakcja zablokowana-wymagana autoryzacja tożsamości" "current_time" : datetime.utcnow().isoformat() } record = json.dumps(record).encode("utf-8") producer.produce(topic=ANOMALY_TOPIC, value=record) producer.flush() elif clients[record['ID']-1,1]==1: prediciton1=second_check.predict(data) prediction2=isolation_second.predict(data) if prediction1[0] == 1 or prediciton2[0] == -1: clients[record['ID']-1,1]=2 _id=str(record["ID"]) record = { "id": _id, "data": "Transakcja zablokowana-wymagana autoryzacja tożsamości" "current_time" : datetime.utcnow().isoformat() } record = json.dumps(record).encode("utf-8") producer.produce(topic=ANOMALY_TOPIC, value=record) producer.flush() else: clients[record['ID']-1,1]=0 _id=str(record["ID"]) record = { "id": _id, "data": "Transakcja OK" "current_time" : datetime.utcnow().isoformat() } record = json.dumps(record).encode("utf-8") producer.produce(topic=ANOMALY_TOPIC, value=record) producer.flush() else: prediciton1=first_check.predict(data) prediction2=isolation_first.predict(data) if prediction1[0] == 0 and prediciton2[0] == 1: _id=str(record["ID"]) record = { "id": _id, "data": "Transakcja OK" "current_time" : datetime.utcnow().isoformat() } record = json.dumps(record).encode("utf-8") producer.produce(topic=ANOMALY_TOPIC, value=record) producer.flush() elif prediction1[0] == and prediciton2[0] == -1: clients[record['ID']-1,1]=1 _id=str(record["ID"]) record = { "id": _id, "data": "Wysłane ostrzeżenie, wymuszenie dodatkowej autoryzacji" "current_time" : datetime.utcnow().isoformat() } record = json.dumps(record).encode("utf-8") producer.produce(topic=ANOMALY_TOPIC, value=record) producer.flush() else: clients[record['ID']-1,1]=1 _id=str(record["ID"]) record = { "id": _id, "data": "Transakcja podejrzana" "current_time" : datetime.utcnow().isoformat() } record = json.dumps(record).encode("utf-8") producer.produce(topic=ANOMALY_TOPIC, value=record) producer.flush() consumer.close()for _ in range(NUM_PARTITIONS): p = Process(target=detekcja_anomalii) p.start()
?
- Assistant
Below are the some flaws and improvements that can be made in the given code:
-
Missing import:
Add the following import statements -
import pandas as pd -
model load typo:
first_check=load("first_check.pkl") second_check=load("second_check.pkl") isolation_first=load("first_isolation.pkl") isolation_second=load("second_isolation.pkl") -
Typo in
TRANSACTOPM_CGvariable, it should beTRANSACTION_CG.TRANSACTION_CG = 'transactions' -
Lack of proper indentation: Some sections of code are not indented properly. For example, under the first if condition in the
detekcja_anomalii()function:if clients[record['ID']-1,1] >= 2: _id=str(record["ID"]) record = { "id": _id, "data": "Transakcja zablokowana-wymagana autoryzacja tożsamości", "current_time" : datetime.utcnow().isoformat() } record = json.dumps(record).encode("utf-8") producer.produce(topic=ANOMALY_TOPIC, value=record) producer.flush() -
JSON object contains invalid keys: Replace
record['ID']withrecord['id']in the if and elif conditions. -
Incorrect indexing in the clients DataFrame: Use
clients.loc[record['id'] - 1, 'Warning']instead ofclients[record['id']-1,1]. -
Missing commas: In some lines, a comma is missing after the "data" key in the JSON object. Add
,after the "data" values. -
In this line:
if prediction1[0] == and prediciton2[0] == -1:Replace
predicate1[0] == and predicate2[0] == -1withpredicate1[0] == 1 and predicate2[0] == -1.
With these changes, the code should work better. However, it is essential to thoroughly test the code to identify any other flaws or improvements needed.
-