Tengo el siguiente codigo python: """ from random import seed, shuffle from agents.judge…
- User
Tengo el siguiente codigo python: """ from random import seed, shuffle from agents.judge import Judge, JudgeOutput
class JudgeConfig(TypedDict): model: str provider: str temperature: float max_tokens: int api_key: str max_retries: Optional[int] = 3
class Pipeline2Config(TypedDict): judge1: JudgeConfig judge2: JudgeConfig judge3: JudgeConfig
class Pipeline2(): def init( self, config:Pipeline2Config, random_seed:int, use_random_seed:bool=False ) -> None: self.judge1 = Judge() self.judge2 = Judge() self.judge3 = Judge()
self.judge1.set_llm( model=config["judge1"]["model"], provider=config["judge1"]["provider"], temperature=config["judge1"]["temperature"], max_tokens=config["judge1"]["max_tokens"], api_key=config["judge1"]["api_key"], max_retries=config["judge1"]["max_retries"] ) self.judge2.set_llm( model=config["judge2"]["model"], provider=config["judge2"]["provider"], temperature=config["judge2"]["temperature"], max_tokens=config["judge2"]["max_tokens"], api_key=config["judge2"]["api_key"], max_retries=config["judge2"]["max_retries"] ) self.judge3.set_llm( model=config["judge3"]["model"], provider=config["judge3"]["provider"], temperature=config["judge3"]["temperature"], max_tokens=config["judge3"]["max_tokens"], api_key=config["judge3"]["api_key"], max_retries=config["judge3"]["max_retries"] ) self.random_seed:int = random_seed self.use_random_seed:bool = use_random_seed def __shuffle_options( self, options:list[dict[str, str]] ) -> tuple[dict[str, str], dict[str, str]]: if self.use_random_seed: seed(self.random_seed) shuffle(options) return options[0], options[1] def run(self, data:pd.DataFrame, max_workers:int=3) -> pd.DataFrame: results_list = [] #! DEBUG ONLY DEBUG ONLY DEBUG ONLY DEBUG ONLY: data = data.head(2).copy() def process_row(idx: int, row: pd.Series) -> pd.DataFrame: question_original:str = row["question_original"] ground_truth:str = row["ground_truth"] translation:str = row["translation"] # We get rid of "<Item>" tags if translation.startswith("<Item>") and translation.endswith("</Item>"): translation:str = translation[7:-8] print("\n-----------------") print(f"ID: {row['id']}") # print("# **Original Question**:") print("# ORIGINAL TEXT") print(question_original) # print("\n# **Ground Truth**") print("\n# Translation 1 (randomized):") print(ground_truth) # print("\n# **Translation**:") print("\n# Translation 2: (randomized)") print(translation) print(f"\n") print("SHUFFLING:") translation_1, translation_2 = self.__shuffle_options([ { "label": "ground_truth", "value": ground_truth }, { "label": "translation", "value": translation }, ]) print("#### TRANSLATION 1:") print(translation_1) print("#### TRANSLATION 2:") print(translation_2) judge1_output:JudgeOutput = self.judge1.get_verdict( source_text=question_original, translation_1=translation_1, translation_2=translation_2 ) judge2_output:JudgeOutput = self.judge2.get_verdict( source_text=question_original, translation_1=translation_1, translation_2=translation_2 ) judge3_output:JudgeOutput = self.judge3.get_verdict( source_text=question_original, translation_1=translation_1, translation_2=translation_2 ) return pd.DataFrame([{ "data": 0 }]) with ThreadPoolExecutor(max_workers=max_workers) as executor: future_to_index = { executor.submit( process_row, idx, row): idx for idx, row in data.iterrows() } for future in as_completed(future_to_index): try: result_df = future.result() results_list.append(result_df) except Exception as e: print(f"Error processing row {future_to_index[future]}: {e}") returnresults:pd.DataFrame = results
RANDOM_SEED:int = 262025
pipeline2 = Pipeline2(config={ "judge1": { "model": "gemini-2.5-flash-preview-05-20", "provider": "google_genai", "temperature": 0.7, "max_tokens": 4096, "max_retries": 3, "api_key": GOOGLE_GENAI_API_KEY }, "judge2": { "model": "gemini-2.5-flash-preview-05-20", "provider": "google_genai", "temperature": 0.7, "max_tokens": 4096, "max_retries": 3, "api_key": GOOGLE_GENAI_API_KEY }, "judge3": { "model": "gemini-2.5-flash-preview-05-20", "provider": "google_genai", "temperature": 0.7, "max_tokens": 4096, "max_retries": 3, "api_key": GOOGLE_GENAI_API_KEY }, }, random_seed=RANDOM_SEED, use_random_seed=True )
pipeline2.run(data=results) """
Necesito editarlo para lograr que las llamadas ".get_verdict()" (que llaman a un LLM a traves de una API) se ejecuten de manera asyncrona para no tener que esperar a que termine el primer "juez" para poder pasar a la siguiente consulta. Finalmente las respuestas deben ser retornadas.
- Assistant
Para lograr que las llamadas a
.get_verdict()se ejecuten de manera asíncrona, puedes utilizar la bibliotecaasyncioy la funciónasyncio.gather()para ejecutar las llamadas de manera concurrente. Sin embargo, dado que la claseJudgey su métodoget_verdict()no están diseñados para ser asíncronos, necesitarás adaptarlos o crear una capa de abstracción que permita la ejecución asíncrona.Aquí te muestro un ejemplo de cómo podrías adaptar tu código para lograr la ejecución asíncrona de las llamadas a
.get_verdict()utilizandoasyncioyasyncio.gather(). Este ejemplo asume queJudgeyget_verdict()pueden ser adaptados para ser asíncronos o que se crea una capa de abstracción para lograrlo.Primero, necesitas importar
asyncioy adaptar tu claseJudgepara que su métodoget_verdict()sea asíncrono. Si no puedes modificar la claseJudge, deberás crear una capa de abstracción que permita la ejecución asíncrona.import asyncio from typing import Dict, List # Adaptación para que get_verdict sea asíncrono class AsyncJudge: def __init__(self, judge: Judge): self.judge = judge async def get_verdict(self, source_text: str, translation_1: Dict, translation_2: Dict) -> JudgeOutput: # Simulación de llamada asíncrona # En realidad, aquí deberías hacer la llamada a la API de manera asíncrona # Si la API no soporta llamadas asíncronas, considera usar threading o multiprocessing loop = asyncio.get_running_loop() return await loop.run_in_executor(None, self.judge.get_verdict, source_text, translation_1, translation_2) class Pipeline2: # ... (resto del código) def __init__(self, config: Pipeline2Config, random_seed: int, use_random_seed: bool = False): # ... (resto del código) self.async_judge1 = AsyncJudge(self.judge1) self.async_judge2 = AsyncJudge(self.judge2) self.async_judge3 = AsyncJudge(self.judge3) async def process_row(self, idx: int, row: pd.Series) -> pd.DataFrame: question_original: str = row["question_original"] ground_truth: str = row["ground_truth"] translation: str = row["translation"] # ... (resto del código) translation_1, translation_2 = self.__shuffle_options([ {"label": "ground_truth", "value": ground_truth}, {"label": "translation", "value": translation}, ]) # Llamadas asíncronas a get_verdict tasks = [ self.async_judge1.get_verdict(question_original, translation_1, translation_2), self.async_judge2.get_verdict(question_original, translation_1, translation_2), self.async_judge3.get_verdict(question_original, translation_1, translation_2), ] results = await asyncio.gather(*tasks) # Procesar los resultados return pd.DataFrame([{"data": 0}]) # Aquí deberías procesar los resultados async def run(self, data: pd.DataFrame, max_workers: int = 3) -> List[pd.DataFrame]: tasks = [] for idx, row in data.iterrows(): task = asyncio.create_task(self.process_row(idx, row)) tasks.append(task) results = await asyncio.gather(*tasks) return results # Ejecución pipeline2 = Pipeline2(config={ # ... (configuración) }, random_seed=RANDOM_SEED, use_random_seed=True) async def main(): results = await pipeline2.run(data=results) # Procesar los resultados asyncio.run(main())Este ejemplo muestra cómo adaptar tu código para ejecutar las llamadas a
.get_verdict()de manera asíncrona. Sin embargo, ten en cuenta que si la API que estás utilizando no soporta llamadas asíncronas, podrías necesitar utilizarthreadingomultiprocessingpara lograr la ejecución concurrente.Recuerda que la clave para la ejecución asíncrona es utilizar
asyncio.gather()para ejecutar las tareas de manera concurrente y esperar a que todas terminen antes de proceder.
Other models, same or similar prompt
Responses were produced in different, uncontrolled contexts (system prompts, settings, prior turns). This is not a benchmark.