Skip to content
The Chat Archive
gpt-4o-2024-08-06· OpenAI WildChat-4.8M (AI2)

This is my current working method: import unittest from unittest.mock import AsyncMock,…

  1. User

    This is my current working method:

    import unittest from unittest.mock import AsyncMock, patch from langchain_core.prompts import PromptTemplate from langchain_core.runnables import RunnableLambda from langchain_core.runnables.retry import RunnableRetry from langchain_openai import ChatOpenAI import os

    Define the mock function for simulating a ValueError

    async def mock_invoke(*args, **kwargs): raise ValueError({ 'message': '{"error":{"code":null,"message":"Rate limit exceeded","param":null,"type":"rate_limit_exceeded"}}\n', 'code': 429 })

    Method 1

    async def test_method_1(): with patch('langchain_openai.ChatOpenAI.invoke', new_callable=AsyncMock) as invoke_mock: # Apply mock function to side effect invoke_mock.side_effect = mock_invoke

        # Define prompt and model
        template = PromptTemplate.from_template("Tell me a joke about {topic}.")
    
        # Add retry logic to the model invocation
        runnable_with_retries = llm.with_retry(
            retry_if_exception_type=(ValueError,),  # Retry only on ValueError exceptions
            wait_exponential_jitter=True,           # Enable jitter for backoff timing
            stop_after_attempt=3                    # Maximum 3 attempts
        )
    
        # Create a chain with the template and the retry-enabled model
        chain = template | runnable_with_retries
    
        # Invoke the chain with input and assert the result
        print("Before invoking chain")
        try:
            await chain.invoke({"topic": "programmers"})
        except ValueError as e:
            print(f"Method 1: Caught expected exception: {e}")
        print("After invoking chain")
    

    Method 2

    async def test_method_2(): with patch('langchain_openai.ChatOpenAI.invoke', new_callable=AsyncMock) as invoke_mock: # Apply mock function to side effect invoke_mock.side_effect = mock_invoke

        # Wrap the model invocation in a Runnable
        runnable = RunnableLambda(lambda input: llm.invoke(input))
    
        # Configure retries
        runnable_with_retries = RunnableRetry(
            bound=runnable,
            retry_exception_types=(ValueError,),   # Retry only on ValueError
            max_attempt_number=3,                  # Retry up to 3 times
            wait_exponential_jitter=True           # Add jitter to backoff
        )
    
        # Invoke the runnable with retries and assert the result
        print("Before invoking runnable_with_retries")
        try:
            await runnable_with_retries.invoke("Tell me a joke.")
        except ValueError as e:
            print(f"Method 2: Caught expected exception: {e}")
        print("After invoking runnable_with_retries")
    

    Run the tests

    await test_method_1() await test_method_2()

    I want to add a way to mock two errors and then follow up with a correct response, how can I do that?

  2. Assistant

    To mock a sequence of responses, where the initial responses are errors and the subsequent one is a success, you can use the side_effect feature of the mock to specify a list of side effects. Here's how you can modify your code to simulate two errors (e.g., ValueError) followed by a successful response on the third attempt:

    import unittest
    from unittest.mock import AsyncMock, patch
    from langchain_core.prompts import PromptTemplate
    from langchain_openai import ChatOpenAI
    import os
    
    # Define the mock function for simulating errors followed by a success
    async def mock_invoke(*args, **kwargs):
        # Simulate a successful response after initial errors
        return {"text": "Why do programmers prefer dark mode? Because light attracts bugs!"}
    
    # Mock sequence: two errors followed by a success
    mock_sequence = [
        ValueError({'message': '{"error":{"code":null,"message":"Rate limit exceeded","param":null,"type":"rate_limit_exceeded"}}\n', 'code': 429}),
        ValueError({'message': '{"error":{"code":null,"message":"Rate limit exceeded","param":null,"type":"rate_limit_exceeded"}}\n', 'code': 429}),
        mock_invoke
    ]
    
    # Method 1
    async def test_method_1():
        with patch('langchain_openai.ChatOpenAI.invoke', new_callable=AsyncMock) as invoke_mock:
            # Apply mock sequence to side effect
            invoke_mock.side_effect = mock_sequence
    
            # Define prompt and model
            template = PromptTemplate.from_template("Tell me a joke about {topic}.")
    
            # Mock placing the llm object since you might need that for actual logic
            llm = ChatOpenAI()
    
            # Add retry logic to the model invocation
            runnable_with_retries = llm.with_retry(
                retry_if_exception_type=(ValueError,),  # Retry only on ValueError exceptions
                wait_exponential_jitter=True,           # Enable jitter for backoff timing
                stop_after_attempt=3                    # Maximum 3 attempts
            )
    
            # Create a chain with the template and the retry-enabled model
            chain = template | runnable_with_retries
    
            # Invoke the chain with input and assert the result
            print("Before invoking chain")
            try:
                result = await chain.invoke({"topic": "programmers"})
                print(f"Method 1: Received expected success result: {result}")
            except ValueError as e:
                print(f"Method 1: Caught unexpected exception: {e}")
            print("After invoking chain")
    
    # Method 2
    async def test_method_2():
        with patch('langchain_openai.ChatOpenAI.invoke', new_callable=AsyncMock) as invoke_mock:
            # Apply mock sequence to side effect
            invoke_mock.side_effect = mock_sequence
    
            # Mock placing the llm object since you might need that for actual logic
            llm = ChatOpenAI()
    
            # Wrap the model invocation in a Runnable
            runnable = RunnableLambda(lambda input: llm.invoke(input))
    
            # Configure retries
            runnable_with_retries = RunnableRetry(
                bound=runnable,
                retry_exception_types=(ValueError,),   # Retry only on ValueError
                max_attempt_number=3,                  # Retry up to 3 times
                wait_exponential_jitter=True           # Add jitter to backoff
            )
    
            # Invoke the runnable with retries and assert the result
            print("Before invoking runnable_with_retries")
            try:
                result = await runnable_with_retries.invoke("Tell me a joke.")
                print(f"Method 2: Received expected success result: {result}")
            except ValueError as e:
                print(f"Method 2: Caught unexpected exception: {e}")
            print("After invoking runnable_with_retries")
    
    # Run the tests
    await test_method_1()
    await test_method_2()
    

    Explanation:

    • mock_sequence: Created a list of side effects, with two ValueError instances followed by a successful response via the mock_invoke function.
    • invoke_mock.side_effect: Assigned this list to the side_effect of your mock object. The mock will raise the ValueError for the first two calls and return a successful result on the third invocation.
    • Test Methods: Updated the test methods to print the expected success result after retries are exhausted.

    This setup will mimic hitting a rate limit twice and succeeding on the third attempt in both methods.

  3. User

    from what I can see from the output, it seems it was only called once, was this because of our mock method or something?

    Before invoking chain Method 1: Caught unexpected exception: {'message': '{"error":{"code":null,"message":"Rate limit exceeded","param":null,"type":"rate_limit_exceeded"}}\n', 'code': 429} After invoking chain Before invoking runnable_with_retries Method 2: Caught unexpected exception: {'message': '{"error":{"code":null,"message":"Rate limit exceeded","param":null,"type":"rate_limit_exceeded"}}\n', 'code': 429} After invoking runnable_with_retries

  4. Assistant

    It seems that the retry mechanism did not function as expected, resulting in the exception being thrown on the first attempt without retrying. Let's go through a few possible reasons and their solutions:

    1. Ensure the Retry Configuration is Set Correctly:

      • Make sure that the runnable_with_retries is correctly configured to handle exceptions and retry upon encountering them.
      • Double-check that the retry mechanism is being correctly applied to your chain and runnable.
    2. Verify LLM Mocking and Integration:

      • Ensure that all components of the original function, including any local variables or setup like the llm object, are correctly mocked.
      • If ChatOpenAI is used in RunnableLambda, ensure it's correctly passed or replaced with a mocked version.
    3. Log the Number of Attempts:

      • Consider adding print statements or logs inside the mock to track how many times the mocked function mock_invoke is being executed. This can assist in verifying if the retry attempts are processed correctly.
    4. Check the Patch Target:

      • Verify that the patch target is correct. The string inside patch should match the actual location of the invoke method being called in your application code path.

    Here's an adjusted approach to verify retry attempts and correct usage of the retry mechanism:

    import unittest
    from unittest.mock import AsyncMock, patch
    from langchain_core.prompts import PromptTemplate
    from langchain_openai import ChatOpenAI
    import asyncio
    
    attempts = 0  # Track the number of retry attempts
    
    # Define the mock function for simulating errors followed by a success
    async def mock_invoke(*args, **kwargs):
        global attempts
        attempts += 1
        if attempts < 3:  # Simulate error for first two calls
            raise ValueError({'message': '{"error":{"code":null,"message":"Rate limit exceeded","param":null,"type":"rate_limit_exceeded"}}\n', 'code': 429})
        return {"text": "Why do programmers prefer dark mode? Because light attracts bugs!"}
    
    # Method 1
    async def test_method_1():
        global attempts
        attempts = 0
        
        with patch('langchain_openai.ChatOpenAI.invoke', new_callable=AsyncMock) as invoke_mock:
            invoke_mock.side_effect = mock_invoke
    
            # Define prompt and model
            template = PromptTemplate.from_template("Tell me a joke about {topic}.")
            
            # Instantiate ChatOpenAI as necessary in this context
            llm = ChatOpenAI()
            
            # Add retry logic to the model invocation
            runnable_with_retries = llm.with_retry(
                retry_if_exception_type=(ValueError, ),
                wait_exponential_jitter=False,  # False or modify as needed for testing
                stop_after_attempt=3
            )
    
            # Chain with template and retry-enabled model
            chain = template | runnable_with_retries
    
            # Invoke and verify retry
            print("Before invoking chain")
            try:
                result = await chain.invoke({"topic": "programmers"})
                print(f"Method 1: Received result: {result}")
            except ValueError as e:
                print(f"Method 1: Unexpected exception: {e}")
            print("After invoking chain")
            print(f"Retries attempted: {attempts}")
    
    # Method 2
    async def test_method_2():
        global attempts
        attempts = 0
        
        with patch('langchain_openai.ChatOpenAI.invoke', new_callable=AsyncMock) as invoke_mock:
            invoke_mock.side_effect = mock_invoke
    
            # Instantiate ChatOpenAI as necessary in this context
            llm = ChatOpenAI()
            
            # Wrap the model invocation in a Runnable
            runnable = RunnableLambda(lambda input: llm.invoke(input))
    
            # Configure retries
            runnable_with_retries = RunnableRetry(
                bound=runnable,
                retry_exception_types=(ValueError, ),
                max_attempt_number=3,
                wait_exponential_jitter=False
            )
    
            # Invoke and verify retry
            print("Before invoking runnable_with_retries")
            try:
                result = await runnable_with_retries.invoke("Tell me a joke.")
                print(f"Method 2: Received result: {result}")
            except ValueError as e:
                print(f"Method 2: Unexpected exception: {e}")
            print("After invoking runnable_with_retries")
            print(f"Retries attempted: {attempts}")
    
    # Run the tests
    await test_method_1()
    await test_method_2()
    

    Explanation:

    • Global attempts Counter: A global variable attempts is used to track the number of attempts made to invoke the mock method.
    • mock_invoke Logic: The mock_invoke function raises a ValueError for the first two calls and returns a success on the third.
    • Patch Verification and Initialization: Ensuring ChatOpenAI() is mocked or instantiated correctly and the retry logic is correctly applied.
    • Disable Jitter for Testing: Temporarily setting wait_exponential_jitter=False allows for easier testing and verification without delay.

    If the retry mechanism isn't being applied, you might need to check elements like the exact integration of with_retry and make sure your patch target within patch() accurately refers to what you're testing.

  5. User

    this was the output:

    Before invoking chain Method 1: Unexpected exception: {'message': '{"error":{"code":null,"message":"Rate limit exceeded","param":null,"type":"rate_limit_exceeded"}}\n', 'code': 429} After invoking chain Retries attempted: 1 Before invoking runnable_with_retries Method 2: Unexpected exception: {'message': '{"error":{"code":null,"message":"Rate limit exceeded","param":null,"type":"rate_limit_exceeded"}}\n', 'code': 429} After invoking runnable_with_retries Retries attempted: 1

  6. Assistant

    It seems that the retry logic might not be working as expected, leading to the ValueError being raised on the first attempt without retrying. Here are some steps to diagnose and fix the issue:

    1. Check Retry Logic Setup:

      • Ensure that both methods (llm.with_retry and RunnableRetry) are set up to handle ValueError exceptions specifically. If these configurations are not correctly applying the retry logic, it could cause the issue you're facing.
    2. Verify Object Instantiation:

      • Ensure the model ChatOpenAI and PromptTemplate are correctly instantiated. If with_retry or RunnableRetry are not designed to handle the mocking structure, it may not work as intended.
    3. Fix Mocking Scope:

      • Ensure the patch target is correctly pointing to where invoke is used in your context.
      • Check whether any wrappers or adapters in with_retry or RunnableRetry might intercede, potentially affecting the retry attempts.
    4. Debugging Steps:

      • Add logging in the mock_invoke and runnable_with_retries to trace execution flows.
      • Check if any try-except blocks around retry mechanisms are inadvertently catching the exceptions, preventing retry logic from kicking in.
    5. Direct Retry Configuration Test:

      • Simplify the test by isolating logic to explicitly confirm that with_retry and RunnableRetry can perform retries as expected when directly instantiated.

    Below is a revised breakdown which ensures simplicity in testing these constructs:

    import unittest
    from unittest.mock import AsyncMock, patch
    from langchain_core.prompts import PromptTemplate
    from langchain_openai import ChatOpenAI
    from langchain_core.runnables import RunnableLambda
    from langchain_core.runnables.retry import RunnableRetry
    import asyncio
    
    attempts = 0  # Track the number of retry attempts
    
    # Define the mock function to simulate two failures followed by success
    async def mock_invoke(*args, **kwargs):
        global attempts
        attempts += 1
        if attempts <= 2:  # Trigger error for the first two invocations
            raise ValueError({
                'message': '{"error":{"code":null,"message":"Rate limit exceeded","param":null,"type":"rate_limit_exceeded"}}\n',
                'code': 429
            })
        return {"text": "Why do programmers prefer dark mode? Because light attracts bugs!"}
    
    # Method 1
    async def test_method_1():
        global attempts
        attempts = 0  # Reset attempt counter at start of test
        
        with patch('langchain_openai.ChatOpenAI.invoke', new_callable=AsyncMock) as invoke_mock:
            invoke_mock.side_effect = mock_invoke
    
            # Define prompt and model
            template = PromptTemplate.from_template("Tell me a joke about {topic}.")
            
            # Direct attempt at using retries with ChatOpenAI
            llm = ChatOpenAI()
            
            runnable_with_retries = llm.with_retry(
                retry_if_exception_type=(ValueError, ),
                wait_exponential_jitter=False,  # Disable jitter to quickly test retries
                stop_after_attempt=3
            )
    
            # Create a chain with the template and the retry-enabled model
            chain = template | runnable_with_retries
    
            print("Before invoking chain")
            try:
                result = await chain.invoke({"topic": "programmers"})
                print(f"Method 1: Received result: {result}")
            except Exception as e:
                print(f"Method 1: Unexpected exception: {e}")
            print("After invoking chain")
            print(f"Retries attempted for Method 1: {attempts}")
    
    # Method 2
    async def test_method_2():
        global attempts
        attempts = 0  # Reset attempt counter at start of test
        
        with patch('langchain_openai.ChatOpenAI.invoke', new_callable=AsyncMock) as invoke_mock:
            invoke_mock.side_effect = mock_invoke
    
            llm = ChatOpenAI()
    
            # Wrap the model invocation in a Runnable
            runnable = RunnableLambda(lambda input: llm.invoke(input))
    
            # Configure retries
            runnable_with_retries = RunnableRetry(
                bound=runnable,
                retry_exception_types=(ValueError, ),
                max_attempt_number=3,
                wait_exponential_jitter=False  # Simplified for testing purposes
            )
    
            print("Before invoking runnable_with_retries")
            try:
                result = await runnable_with_retries.invoke("Tell me a joke.")
                print(f"Method 2: Received result: {result}")
            except Exception as e:
                print(f"Method 2: Unexpected exception: {e}")
            print("After invoking runnable_with_retries")
            print(f"Retries attempted for Method 2: {attempts}")
    
    # Run the tests
    asyncio.run(test_method_1())
    asyncio.run(test_method_2())
    

    Key Adjustments:

    • Ensure Mock Setup Properly Controls Call Conditions: The mock is set to simulate two failed attempts followed by success, directly replaying the scenario of retry.
    • Logging Attempts: I added logging indicators to monitor retry attempts specifically, verifying if the retry mechanism is correctly incrementing.
    • Direct Testing of Callable Retry Mechanism: This scenario tests your retry classes in isolation to ensure they're executing.

    This setup should reliably test whether retries are being executed through either of the retry policies (with_retry or RunnableRetry).

  7. User

    it is not working but now I can understand why,if we change the invoke method, it won't call the original one, maybe this is interfering with how we set up the code

  8. Assistant

    Given your understanding that the retry logic might not be correctly configured due to how the mocking interferes with the original method call, let's take a deep dive into how to properly integrate mocking with retry logic.

    Here's a detailed plan to help isolate the retry logic for successful integration with mock tests:

    1. Ensure Mocking Targets the Correct Method: This involves ensuring that you correctly target the function where the invoke method is actually called in your workflow or codebase, i.e., where invoke is originally utilized in ChatOpenAI.

    2. Recreate Retry Configuration if Needed: If retry logic is custom or otherwise dependent on specific handling, ensure that your mock retains the required callable chain correctly through retry attempts.

    3. Use Internal Mocking Methods: Instead of complex setups, use Python's inherent mocking capabilities to control retries.

    Here’s a focused example demonstrating how one might correctly configure and test retry logic using mocking:

    import unittest
    from unittest.mock import AsyncMock, patch
    from langchain_core.prompts import PromptTemplate
    from langchain_openai import ChatOpenAI
    from langchain_core.runnables import RunnableLambda
    from langchain_core.runnables.retry import RunnableRetry
    import asyncio
    
    # Global attempt counters
    attempts_method_1 = 0
    attempts_method_2 = 0
    
    # Mock invoke function with retry logic incorporated
    async def mock_successful_invoke(*args, **kwargs):
        return {"text": "Why do programmers prefer dark mode? Because light attracts bugs!"}
    
    async def mock_failing_invoke(*args, **kwargs):
        raise ValueError({
            'message': '{"error":{"code":null,"message":"Rate limit exceeded","param":null,"type":"rate_limit_exceeded"}}\n',
            'code': 429
        })
    
    # Testing Method 1
    async def test_method_1():
        global attempts_method_1
        attempts_method_1 = 0
    
        with patch.object(ChatOpenAI, 'invoke', new_callable=AsyncMock) as invoke_mock:
            # Setup mock side effects
            invoke_mock.side_effect = [mock_failing_invoke(), mock_failing_invoke(), mock_successful_invoke()]
    
            template = PromptTemplate.from_template("Tell me a joke about {topic}.")
            llm = ChatOpenAI()
    
            # Create retry-enabled invoke
            runnable_with_retries = llm.with_retry(
                retry_if_exception_type=(ValueError, ),
                wait_exponential_jitter=False,
                stop_after_attempt=3
            )
    
            chain = template | runnable_with_retries
    
            print("Before invoking chain")
            try:
                result = await chain.invoke({"topic": "programmers"})
                print(f"Method 1: Received result: {result}")
            except ValueError as e:
                print(f"Method 1: Caught exception: {e}")
            print("After invoking chain")
            print(f"Retries attempted for Method 1: {invoke_mock.call_count}")
    
    # Testing Method 2
    async def test_method_2():
        global attempts_method_2
        attempts_method_2 = 0
    
        with patch.object(ChatOpenAI, 'invoke', new_callable=AsyncMock) as invoke_mock:
            # Setup mock side effects
            invoke_mock.side_effect = [mock_failing_invoke(), mock_failing_invoke(), mock_successful_invoke()]
    
            llm = ChatOpenAI()
    
            runnable = RunnableLambda(lambda input: llm.invoke(input))
    
            runnable_with_retries = RunnableRetry(
                bound=runnable,
                retry_exception_types=(ValueError, ),
                max_attempt_number=3,
                wait_exponential_jitter=False
            )
    
            print("Before invoking runnable_with_retries")
            try:
                result = await runnable_with_retries.invoke("Tell me a joke.")
                print(f"Method 2: Received result: {result}")
            except ValueError as e:
                print(f"Method 2: Caught exception: {e}")
            print("After invoking runnable_with_retries")
            print(f"Retries attempted for Method 2: {invoke_mock.call_count}")
    
    # Run the tests
    asyncio.run(test_method_1())
    asyncio.run(test_method_2())
    

    Key Changes:

    • Mock Object Configuration using Object Method: Uses patch.object targeting the ChatOpenAI class to more reliably intercept invoke method calls, ensuring full control over its behavior.

    • Defined side_effect List: Clearly defined side_effect list with successive failures followed by success allows easy tracing of how the retry logic reacts.

    • Relying on Mock's call_count: Instead of manual attempts counting, leveraging invoke_mock.call_count provides accurate insight into how many attempts were made, helping diagnose any discrepancies in retry handling.

    This setup streamlines the testing of retry logic by reflecting a typical sequence involving repeated request attempts, where failovers gracefully lead to a successful outcome after exhaustion of prior retry logic.

  9. User

    it gave me this output:

    Before invoking chain Method 1: Received result: <coroutine object mock_failing_invoke at 0x00000231B15B49E0> After invoking chain Retries attempted for Method 1: 1 Before invoking runnable_with_retries Method 2: Received result: <coroutine object mock_failing_invoke at 0x00000231B04DA340> After invoking runnable_with_retries Retries attempted for Method 2: 1

  10. Assistant

    The issue here appears to be that the side_effect list contains coroutine objects rather than their execution results. When you're using coroutines with side effects in Python's unittest.mock, it's crucial to ensure that the actual calls await these coroutine functions. Let's adjust the implementation to correctly handle the asynchronous behavior and evaluate the execution of mock_failing_invoke and mock_successful_invoke.

    Here's how you can fix this:

    1. Properly Await Coroutines in the Side Effect: To ensure the coroutines are awaited during testing, you'll need to define them in a way that fits into the side effect call chain or await them as they are executed.

    2. Adjust the side_effect List Entries: Instead of placing coroutines directly in the list, wrap them so they execute correctly upon being called.

    Here's an updated example, fixing this behavior:

    import unittest
    from unittest.mock import AsyncMock, patch
    from langchain_core.prompts import PromptTemplate
    from langchain_openai import ChatOpenAI
    from langchain_core.runnables import RunnableLambda
    from langchain_core.runnables.retry import RunnableRetry
    import asyncio
    
    # Mock invoke function for simulating async responses
    async def mock_failing_invoke(*args, **kwargs):
        raise ValueError({
            'message': '{"error":{"code":null,"message":"Rate limit exceeded","param":null,"type":"rate_limit_exceeded"}}\n',
            'code': 429
        })
    
    async def mock_successful_invoke(*args, **kwargs):
        return {"text": "Why do programmers prefer dark mode? Because light attracts bugs!"}
    
    # Testing Method 1
    async def test_method_1():
        with patch.object(ChatOpenAI, 'invoke', new_callable=AsyncMock) as invoke_mock:
            # Setup mock to call the coroutine functions
            invoke_mock.side_effect = [
                mock_failing_invoke,  # Call the function itself, which will be awaited
                mock_failing_invoke,
                mock_successful_invoke
            ]
    
            template = PromptTemplate.from_template("Tell me a joke about {topic}.")
            llm = ChatOpenAI()
    
            # Create retry-enabled invoke
            runnable_with_retries = llm.with_retry(
                retry_if_exception_type=(ValueError, ),
                wait_exponential_jitter=False,
                stop_after_attempt=3
            )
    
            chain = template | runnable_with_retries
    
            print("Before invoking chain")
            try:
                result = await chain.invoke({"topic": "programmers"})
                print(f"Method 1: Received result: {result}")
            except ValueError as e:
                print(f"Method 1: Caught expected exception: {e}")
            print("After invoking chain")
            print(f"Retries attempted for Method 1: {invoke_mock.call_count}")
    
    # Testing Method 2
    async def test_method_2():
        with patch.object(ChatOpenAI, 'invoke', new_callable=AsyncMock) as invoke_mock:
            # Setup mock to call the coroutine functions
            invoke_mock.side_effect = [
                mock_failing_invoke,
                mock_failing_invoke,
                mock_successful_invoke
            ]
    
            llm = ChatOpenAI()
    
            runnable = RunnableLambda(lambda input: llm.invoke(input))
    
            runnable_with_retries = RunnableRetry(
                bound=runnable,
                retry_exception_types=(ValueError, ),
                max_attempt_number=3,
                wait_exponential_jitter=False
            )
    
            print("Before invoking runnable_with_retries")
            try:
                result = await runnable_with_retries.invoke("Tell me a joke.")
                print(f"Method 2: Received result: {result}")
            except ValueError as e:
                print(f"Method 2: Caught expected exception: {e}")
            print("After invoking runnable_with_retries")
            print(f"Retries attempted for Method 2: {invoke_mock.call_count}")
    
    # Run the tests
    asyncio.run(test_method_1())
    asyncio.run(test_method_2())
    

    Key Adjustments:

    • side_effect Entries: The functions in side_effect are specified as functions (not coroutine objects) which invoke_mock will call and properly await during the mock invocation.

    • Awaiting Outcomes: When invoke is called, the mock will execute these functions, ensuring they are awaited correctly as part of the invocation process.

    This approach ensures asynchronous actions are correctly handled, counting retries correctly across the different stages of a mocked function call.

  11. User

    same output:

    Before invoking chain Method 1: Received result: <function mock_failing_invoke at 0x00000231B12772E0> After invoking chain Retries attempted for Method 1: 1 Before invoking runnable_with_retries Method 2: Received result: <function mock_failing_invoke at 0x00000231B12772E0> After invoking runnable_with_retries Retries attempted for Method 2: 1

  12. Assistant

    It seems like there was a misunderstanding in setting up the side effects to correctly simulate a sequence of failures followed by a success. The side effects need to be coroutines when you're using async mocks, which means you should be returning the actual coroutine for each step in the side effect sequence. When you define a mock side effect in an async context, each entry should be an awaitable function.

    Let's correct this by using lambda functions to return the coroutine as part of the side effect list:

    import unittest
    from unittest.mock import AsyncMock, patch
    from langchain_core.prompts import PromptTemplate
    from langchain_openai import ChatOpenAI
    from langchain_core.runnables import RunnableLambda
    from langchain_core.runnables.retry import RunnableRetry
    import asyncio
    
    async def mock_failing_invoke(*args, **kwargs):
        raise ValueError({
            'message': '{"error":{"code":null,"message":"Rate limit exceeded","param":null,"type":"rate_limit_exceeded"}}\n',
            'code': 429
        })
    
    async def mock_successful_invoke(*args, **kwargs):
        return {"text": "Why do programmers prefer dark mode? Because light attracts bugs!"}
    
    # Testing Method 1
    async def test_method_1():
        with patch.object(ChatOpenAI, 'invoke', new_callable=AsyncMock) as invoke_mock:
            # Setup mock behavior with lambda to correctly await coroutines
            invoke_mock.side_effect = [
                lambda *args, **kwargs: mock_failing_invoke(*args, **kwargs),
                lambda *args, **kwargs: mock_failing_invoke(*args, **kwargs),
                lambda *args, **kwargs: mock_successful_invoke(*args, **kwargs)
            ]
    
            template = PromptTemplate.from_template("Tell me a joke about {topic}.")
            llm = ChatOpenAI()
    
            # Create retry-enabled invoke
            runnable_with_retries = llm.with_retry(
                retry_if_exception_type=(ValueError, ),
                wait_exponential_jitter=False,
                stop_after_attempt=3
            )
    
            chain = template | runnable_with_retries
    
            print("Before invoking chain")
            try:
                result = await chain.invoke({"topic": "programmers"})
                print(f"Method 1: Received result: {result}")
            except ValueError as e:
                print(f"Method 1: Caught expected exception: {e}")
            print("After invoking chain")
            print(f"Retries attempted for Method 1: {invoke_mock.call_count}")
    
    # Testing Method 2
    async def test_method_2():
        with patch.object(ChatOpenAI, 'invoke', new_callable=AsyncMock) as invoke_mock:
            # Setup mock behavior with lambda to correctly await coroutines
            invoke_mock.side_effect = [
                lambda *args, **kwargs: mock_failing_invoke(*args, **kwargs),
                lambda *args, **kwargs: mock_failing_invoke(*args, **kwargs),
                lambda *args, **kwargs: mock_successful_invoke(*args, **kwargs)
            ]
    
            llm = ChatOpenAI()
    
            runnable = RunnableLambda(lambda input: llm.invoke(input))
    
            runnable_with_retries = RunnableRetry(
                bound=runnable,
                retry_exception_types=(ValueError, ),
                max_attempt_number=3,
                wait_exponential_jitter=False
            )
    
            print("Before invoking runnable_with_retries")
            try:
                result = await runnable_with_retries.invoke("Tell me a joke.")
                print(f"Method 2: Received result: {result}")
            except ValueError as e:
                print(f"Method 2: Caught expected exception: {e}")
            print("After invoking runnable_with_retries")
            print(f"Retries attempted for Method 2: {invoke_mock.call_count}")
    
    # Run the tests
    asyncio.run(test_method_1())
    asyncio.run(test_method_2())
    

    Explanation of Changes:

    • Lambda Functions: Wrapped each coroutine call within a lambda function. This ensures that when the invoke_mock uses these side effects, it correctly invokes each coroutine. By defining lambdas, the function is not executed immediately but is instead returned to be called later, allowing the async nature to be preserved.

    • Correctly Set Up Async Await: This approach ensures the functions being run are actually awaited, simulating the retry mechanism by counting attempts correctly and proceeding through the side effects as intended.

    This should help verify that your retry mechanism correctly attempts each call.

  13. User

    output:

    Before invoking chain Method 1: Received result: <function test_method_1.. at 0x00000231B1277D80> After invoking chain Retries attempted for Method 1: 1 Before invoking runnable_with_retries Method 2: Received result: <function test_method_2.. at 0x00000231B1277880> After invoking runnable_with_retries Retries attempted for Method 2: 1

    So it didn't changed much, maybe the problem is in the original llm object

  14. Assistant

    It seems that the retry mechanism isn't functioning as expected, and the issue might be related to the retry configuration itself or how the LangChain objects are handling retries. Let's take a step-by-step approach to ensure that everything is correctly configured.

    Key Steps

    1. Check Retry Configuration: Ensure your with_retry method and RunnableRetry objects are correctly configured to handle retries. Ensure they're correctly connected to the parts of the code where exceptions might occur.

    2. Mock Correct Object/Method: Verify the exact path of the method being mocked with patch. Ensure it reflects how and where the invoke method is being used in the LangChain's scope.

    3. Log Actual Calls: You can log what methods get called, effectively serving as a debug to ensure what's being hit during retries.

    4. Isolate Retry Logic: Simplify the test or simulate a retry function to ascertain whether the retry logic itself behaves correctly.

    5. Review Object Construction: Given that llm (likely ChatOpenAI) is throughout the LangChain ecosystem, confirm that the instantiation (ChatOpenAI()) correctly points to proper configurations that are intercepted by your mock.

    Below is a revised example that sets the lambda side effects directly, ensuring execution relevance for each step in asynchronous execution:

    import unittest
    from unittest.mock import AsyncMock, patch
    from langchain_core.prompts import PromptTemplate
    from langchain_openai import ChatOpenAI
    from langchain_core.runnables import RunnableLambda
    from langchain_core.runnables.retry import RunnableRetry
    import asyncio
    
    # Mock invoke function simulating async responses
    async def mock_failing_invoke(*args, **kwargs):
        raise ValueError({
            'message': '{"error":{"code":null,"message":"Rate limit exceeded","param":null,"type":"rate_limit_exceeded"}}\n',
            'code': 429
        })
    
    async def mock_successful_invoke(*args, **kwargs):
        return {"text": "Why do programmers prefer dark mode? Because light attracts bugs!"}
    
    # Testing Method 1
    async def test_method_1():
        with patch.object(ChatOpenAI, 'invoke', new_callable=AsyncMock) as invoke_mock:
            # Setup mock behavior with awaiting lambdas
            invoke_mock.side_effect = [
                lambda *args, **kwargs: mock_failing_invoke(),
                lambda *args, **kwargs: mock_failing_invoke(),
                lambda *args, **kwargs: mock_successful_invoke()
            ]
    
            template = PromptTemplate.from_template("Tell me a joke about {topic}.")
            llm = ChatOpenAI()
    
            # Create retry-enabled invoke
            runnable_with_retries = llm.with_retry(
                retry_if_exception_type=(ValueError, ),
                wait_exponential_jitter=False,
                stop_after_attempt=3
            )
    
            chain = template | runnable_with_retries
    
            print("Before invoking chain")
            try:
                result = await chain.invoke({"topic": "programmers"})
                print(f"Method 1: Received result: {result}")
            except ValueError as e:
                print(f"Method 1: Caught expected exception: {e}")
            print("After invoking chain")
            print(f"Retries attempted for Method 1: {invoke_mock.call_count}")
    
    # Testing Method 2
    async def test_method_2():
        with patch.object(ChatOpenAI, 'invoke', new_callable=AsyncMock) as invoke_mock:
            # Setup mock behavior with awaiting lambdas
            invoke_mock.side_effect = [
                lambda *args, **kwargs: mock_failing_invoke(),
                lambda *args, **kwargs: mock_failing_invoke(),
                lambda *args, **kwargs: mock_successful_invoke()
            ]
    
            llm = ChatOpenAI()
    
            runnable = RunnableLambda(lambda input: llm.invoke(input))
    
            runnable_with_retries = RunnableRetry(
                bound=runnable,
                retry_exception_types=(ValueError,),
                max_attempt_number=3,
                wait_exponential_jitter=False
            )
    
            print("Before invoking runnable_with_retries")
            try:
                result = await runnable_with_retries.invoke("Tell me a joke.")
                print(f"Method 2: Received result: {result}")
            except ValueError as e:
                print(f"Method 2: Caught expected exception: {e}")
            print("After invoking runnable_with_retries")
            print(f"Retries attempted for Method 2: {invoke_mock.call_count}")
    
    # Run the tests
    asyncio.run(test_method_1())
    asyncio.run(test_method_2())
    

    Points of Consideration:

    • Ensure Valid Path: Verify that patch.object is pointing at the correct class and method of invoke.
    • Consistent Lambda Use: Use lambda functions that correctly call awaited coroutine functions without ambiguity.
    • Validate llm Configuration: Double-check runnable_with_retries and similar elements to ensure they enact retry logic.
    • Logging Internals: Add debug outputs within retry sections to trace what's being awaited and handled.

    This approach aims to ensure testing covers a full retry cycle robustly within the LangChain framework.