Skip to content
TheChatArchiveThe Chat Archive
gpt-4.1-2025-04-14· OpenAIHugging Face LMArena Human Preference 140K

Tengo el siguiente codigo python: """ from random import seed, shuffle from agents.judge…

From a Chatbot Arena battleThe same person sent this prompt to two anonymous models at once. Compare it with Metallama-3.3-70b-instruct’s answer.
  1. User

    Tengo el siguiente codigo python: """ from random import seed, shuffle from agents.judge import Judge, JudgeOutput

    class JudgeConfig(TypedDict): model: str provider: str temperature: float max_tokens: int api_key: str max_retries: Optional[int] = 3

    class Pipeline2Config(TypedDict): judge1: JudgeConfig judge2: JudgeConfig judge3: JudgeConfig

    class Pipeline2(): def init( self, config:Pipeline2Config, random_seed:int, use_random_seed:bool=False ) -> None: self.judge1 = Judge() self.judge2 = Judge() self.judge3 = Judge()

        self.judge1.set_llm(
            model=config["judge1"]["model"],
            provider=config["judge1"]["provider"],
            temperature=config["judge1"]["temperature"],
            max_tokens=config["judge1"]["max_tokens"],
            api_key=config["judge1"]["api_key"],
            max_retries=config["judge1"]["max_retries"]
        )
        self.judge2.set_llm(
            model=config["judge2"]["model"],
            provider=config["judge2"]["provider"],
            temperature=config["judge2"]["temperature"],
            max_tokens=config["judge2"]["max_tokens"],
            api_key=config["judge2"]["api_key"],
            max_retries=config["judge2"]["max_retries"]
        )
        self.judge3.set_llm(
            model=config["judge3"]["model"],
            provider=config["judge3"]["provider"],
            temperature=config["judge3"]["temperature"],
            max_tokens=config["judge3"]["max_tokens"],
            api_key=config["judge3"]["api_key"],
            max_retries=config["judge3"]["max_retries"]
        )
        self.random_seed:int = random_seed
        self.use_random_seed:bool = use_random_seed
    
    def __shuffle_options(
            self,
            options:list[dict[str, str]]
        ) -> tuple[dict[str, str], dict[str, str]]:
        if self.use_random_seed:
            seed(self.random_seed)
        shuffle(options)
        return options[0], options[1]
    
    
    def run(self, data:pd.DataFrame, max_workers:int=3) -> pd.DataFrame:
        results_list = []
        #! DEBUG ONLY DEBUG ONLY DEBUG ONLY DEBUG ONLY:
        data = data.head(2).copy()
    
        def process_row(idx: int, row: pd.Series) -> pd.DataFrame:
            question_original:str = row["question_original"]
            ground_truth:str = row["ground_truth"]
            translation:str = row["translation"]
            # We get rid of "<Item>" tags
            if translation.startswith("<Item>") and translation.endswith("</Item>"):
                translation:str = translation[7:-8]
    
            print("\n-----------------")
            print(f"ID: {row['id']}")
            # print("# **Original Question**:")
            print("# ORIGINAL TEXT")
            print(question_original)
            # print("\n# **Ground Truth**")
            print("\n# Translation 1 (randomized):")
            print(ground_truth)
            # print("\n# **Translation**:")
            print("\n# Translation 2: (randomized)")
            print(translation)
            print(f"\n")
            print("SHUFFLING:")
            translation_1, translation_2 = self.__shuffle_options([
                {
                    "label": "ground_truth",
                    "value": ground_truth
                },
                {
                    "label": "translation",
                    "value": translation
                },
            ])
            print("#### TRANSLATION 1:")
            print(translation_1)
            print("#### TRANSLATION 2:")
            print(translation_2)
            judge1_output:JudgeOutput = self.judge1.get_verdict(
                source_text=question_original,
                translation_1=translation_1,
                translation_2=translation_2
            )
            judge2_output:JudgeOutput = self.judge2.get_verdict(
                source_text=question_original,
                translation_1=translation_1,
                translation_2=translation_2
            )
            judge3_output:JudgeOutput = self.judge3.get_verdict(
                source_text=question_original,
                translation_1=translation_1,
                translation_2=translation_2
            )
            return pd.DataFrame([{
                "data": 0
            }])
    
        with ThreadPoolExecutor(max_workers=max_workers) as executor:
            future_to_index = {
                executor.submit(
                    process_row,
                    idx,
                    row): idx for idx, row in data.iterrows()
            }
    
            for future in as_completed(future_to_index):
                try:
                    result_df = future.result()
                    results_list.append(result_df)
                except Exception as e:
                    print(f"Error processing row {future_to_index[future]}: {e}")
        return
    

    results:pd.DataFrame = results

    RANDOM_SEED:int = 262025

    pipeline2 = Pipeline2(config={ "judge1": { "model": "gemini-2.5-flash-preview-05-20", "provider": "google_genai", "temperature": 0.7, "max_tokens": 4096, "max_retries": 3, "api_key": GOOGLE_GENAI_API_KEY }, "judge2": { "model": "gemini-2.5-flash-preview-05-20", "provider": "google_genai", "temperature": 0.7, "max_tokens": 4096, "max_retries": 3, "api_key": GOOGLE_GENAI_API_KEY }, "judge3": { "model": "gemini-2.5-flash-preview-05-20", "provider": "google_genai", "temperature": 0.7, "max_tokens": 4096, "max_retries": 3, "api_key": GOOGLE_GENAI_API_KEY }, }, random_seed=RANDOM_SEED, use_random_seed=True )

    pipeline2.run(data=results) """

    Necesito editarlo para lograr que las llamadas ".get_verdict()" (que llaman a un LLM a traves de una API) se ejecuten de manera asyncrona para no tener que esperar a que termine el primer "juez" para poder pasar a la siguiente consulta. Finalmente las respuestas deben ser retornadas.

  2. Assistant

    ¡Perfecto! Lo que necesitas es paralelizar las llamadas a .get_verdict() de los tres jueces dentro de cada fila. Actualmente, usas ThreadPoolExecutor para procesar filas en paralelo, pero dentro de cada fila las llamadas a los jueces son secuenciales.

    Solución: Usa concurrent.futures.ThreadPoolExecutor dentro de process_row

    Puedes anidar un ThreadPoolExecutor dentro de process_row para lanzar las tres llamadas a los jueces en paralelo.
    Esto es seguro porque las llamadas a la API de LLM suelen ser I/O bound (no CPU bound), así que los threads funcionan bien.


    Cambios en tu función process_row

    from concurrent.futures import ThreadPoolExecutor, as_completed
    
    def process_row(idx: int, row: pd.Series) -> pd.DataFrame:
        question_original:str = row["question_original"]
        ground_truth:str = row["ground_truth"]
        translation:str = row["translation"]
        if translation.startswith("<Item>") and translation.endswith("</Item>"):
            translation:str = translation[7:-8]
    
        translation_1, translation_2 = self.__shuffle_options([
            {"label": "ground_truth", "value": ground_truth},
            {"label": "translation", "value": translation},
        ])
    
        # Llamadas a los jueces en paralelo
        with ThreadPoolExecutor(max_workers=3) as judge_executor:
            future_judge1 = judge_executor.submit(
                self.judge1.get_verdict,
                source_text=question_original,
                translation_1=translation_1,
                translation_2=translation_2
            )
            future_judge2 = judge_executor.submit(
                self.judge2.get_verdict,
                source_text=question_original,
                translation_1=translation_1,
                translation_2=translation_2
            )
            future_judge3 = judge_executor.submit(
                self.judge3.get_verdict,
                source_text=question_original,
                translation_1=translation_1,
                translation_2=translation_2
            )
            # Espera a que terminen todos
            judge1_output = future_judge1.result()
            judge2_output = future_judge2.result()
            judge3_output = future_judge3.result()
    
        # Retorna los resultados como desees
        return pd.DataFrame([{
            "id": row["id"],
            "judge1_output": judge1_output,
            "judge2_output": judge2_output,
            "judge3_output": judge3_output,
            # ... otros campos que quieras
        }])
    

    Cambios en el run (no necesitas cambiar nada aquí, ya que ya usas ThreadPoolExecutor para filas)


    Resumen

    • Cada fila se procesa en paralelo (como ya tienes).
    • Dentro de cada fila, las llamadas a los tres jueces se hacen en paralelo usando otro ThreadPoolExecutor.
    • Así, aprovechas al máximo la concurrencia para llamadas a la API.

    Código Final (fragmento relevante)

    def process_row(idx: int, row: pd.Series) -> pd.DataFrame:
        question_original:str = row["question_original"]
        ground_truth:str = row["ground_truth"]
        translation:str = row["translation"]
        if translation.startswith("<Item>") and translation.endswith("</Item>"):
            translation:str = translation[7:-8]
    
        translation_1, translation_2 = self.__shuffle_options([
            {"label": "ground_truth", "value": ground_truth},
            {"label": "translation", "value": translation},
        ])
    
        with ThreadPoolExecutor(max_workers=3) as judge_executor:
            futures = [
                judge_executor.submit(self.judge1.get_verdict, source_text=question_original, translation_1=translation_1, translation_2=translation_2),
                judge_executor.submit(self.judge2.get_verdict, source_text=question_original, translation_1=translation_1, translation_2=translation_2),
                judge_executor.submit(self.judge3.get_verdict, source_text=question_original, translation_1=translation_1, translation_2=translation_2),
            ]
            judge1_output = futures[0].result()
            judge2_output = futures[1].result()
            judge3_output = futures[2].result()
    
        return pd.DataFrame([{
            "id": row["id"],
            "judge1_output": judge1_output,
            "judge2_output": judge2_output,
            "judge3_output": judge3_output,
        }])
    

    ¿Quieres que te ayude a adaptar el resto del código para que el DataFrame final tenga los resultados de los jueces? ¿O necesitas que te explique cómo hacerlo usando asyncio (si tus .get_verdict soportan async)?

Responses were produced in different, uncontrolled contexts (system prompts, settings, prior turns). This is not a benchmark.