This is my current working method: import unittest from unittest.mock import AsyncMock,…
- User
This is my current working method:
import unittest from unittest.mock import AsyncMock, patch from langchain_core.prompts import PromptTemplate from langchain_core.runnables import RunnableLambda from langchain_core.runnables.retry import RunnableRetry from langchain_openai import ChatOpenAI import os
Define the mock function for simulating a ValueError
async def mock_invoke(*args, **kwargs): raise ValueError({ 'message': '{"error":{"code":null,"message":"Rate limit exceeded","param":null,"type":"rate_limit_exceeded"}}\n', 'code': 429 })
Method 1
async def test_method_1(): with patch('langchain_openai.ChatOpenAI.invoke', new_callable=AsyncMock) as invoke_mock: # Apply mock function to side effect invoke_mock.side_effect = mock_invoke
# Define prompt and model template = PromptTemplate.from_template("Tell me a joke about {topic}.") # Add retry logic to the model invocation runnable_with_retries = llm.with_retry( retry_if_exception_type=(ValueError,), # Retry only on ValueError exceptions wait_exponential_jitter=True, # Enable jitter for backoff timing stop_after_attempt=3 # Maximum 3 attempts ) # Create a chain with the template and the retry-enabled model chain = template | runnable_with_retries # Invoke the chain with input and assert the result print("Before invoking chain") try: await chain.invoke({"topic": "programmers"}) except ValueError as e: print(f"Method 1: Caught expected exception: {e}") print("After invoking chain")Method 2
async def test_method_2(): with patch('langchain_openai.ChatOpenAI.invoke', new_callable=AsyncMock) as invoke_mock: # Apply mock function to side effect invoke_mock.side_effect = mock_invoke
# Wrap the model invocation in a Runnable runnable = RunnableLambda(lambda input: llm.invoke(input)) # Configure retries runnable_with_retries = RunnableRetry( bound=runnable, retry_exception_types=(ValueError,), # Retry only on ValueError max_attempt_number=3, # Retry up to 3 times wait_exponential_jitter=True # Add jitter to backoff ) # Invoke the runnable with retries and assert the result print("Before invoking runnable_with_retries") try: await runnable_with_retries.invoke("Tell me a joke.") except ValueError as e: print(f"Method 2: Caught expected exception: {e}") print("After invoking runnable_with_retries")Run the tests
await test_method_1() await test_method_2()
I want to add a way to mock two errors and then follow up with a correct response, how can I do that?
- Assistant
To mock a sequence of responses, where the initial responses are errors and the subsequent one is a success, you can use the
side_effectfeature of the mock to specify a list of side effects. Here's how you can modify your code to simulate two errors (e.g.,ValueError) followed by a successful response on the third attempt:import unittest from unittest.mock import AsyncMock, patch from langchain_core.prompts import PromptTemplate from langchain_openai import ChatOpenAI import os # Define the mock function for simulating errors followed by a success async def mock_invoke(*args, **kwargs): # Simulate a successful response after initial errors return {"text": "Why do programmers prefer dark mode? Because light attracts bugs!"} # Mock sequence: two errors followed by a success mock_sequence = [ ValueError({'message': '{"error":{"code":null,"message":"Rate limit exceeded","param":null,"type":"rate_limit_exceeded"}}\n', 'code': 429}), ValueError({'message': '{"error":{"code":null,"message":"Rate limit exceeded","param":null,"type":"rate_limit_exceeded"}}\n', 'code': 429}), mock_invoke ] # Method 1 async def test_method_1(): with patch('langchain_openai.ChatOpenAI.invoke', new_callable=AsyncMock) as invoke_mock: # Apply mock sequence to side effect invoke_mock.side_effect = mock_sequence # Define prompt and model template = PromptTemplate.from_template("Tell me a joke about {topic}.") # Mock placing the llm object since you might need that for actual logic llm = ChatOpenAI() # Add retry logic to the model invocation runnable_with_retries = llm.with_retry( retry_if_exception_type=(ValueError,), # Retry only on ValueError exceptions wait_exponential_jitter=True, # Enable jitter for backoff timing stop_after_attempt=3 # Maximum 3 attempts ) # Create a chain with the template and the retry-enabled model chain = template | runnable_with_retries # Invoke the chain with input and assert the result print("Before invoking chain") try: result = await chain.invoke({"topic": "programmers"}) print(f"Method 1: Received expected success result: {result}") except ValueError as e: print(f"Method 1: Caught unexpected exception: {e}") print("After invoking chain") # Method 2 async def test_method_2(): with patch('langchain_openai.ChatOpenAI.invoke', new_callable=AsyncMock) as invoke_mock: # Apply mock sequence to side effect invoke_mock.side_effect = mock_sequence # Mock placing the llm object since you might need that for actual logic llm = ChatOpenAI() # Wrap the model invocation in a Runnable runnable = RunnableLambda(lambda input: llm.invoke(input)) # Configure retries runnable_with_retries = RunnableRetry( bound=runnable, retry_exception_types=(ValueError,), # Retry only on ValueError max_attempt_number=3, # Retry up to 3 times wait_exponential_jitter=True # Add jitter to backoff ) # Invoke the runnable with retries and assert the result print("Before invoking runnable_with_retries") try: result = await runnable_with_retries.invoke("Tell me a joke.") print(f"Method 2: Received expected success result: {result}") except ValueError as e: print(f"Method 2: Caught unexpected exception: {e}") print("After invoking runnable_with_retries") # Run the tests await test_method_1() await test_method_2()Explanation:
mock_sequence: Created a list of side effects, with twoValueErrorinstances followed by a successful response via themock_invokefunction.invoke_mock.side_effect: Assigned this list to theside_effectof your mock object. The mock will raise theValueErrorfor the first two calls and return a successful result on the third invocation.- Test Methods: Updated the test methods to print the expected success result after retries are exhausted.
This setup will mimic hitting a rate limit twice and succeeding on the third attempt in both methods.
- User
from what I can see from the output, it seems it was only called once, was this because of our mock method or something?
Before invoking chain Method 1: Caught unexpected exception: {'message': '{"error":{"code":null,"message":"Rate limit exceeded","param":null,"type":"rate_limit_exceeded"}}\n', 'code': 429} After invoking chain Before invoking runnable_with_retries Method 2: Caught unexpected exception: {'message': '{"error":{"code":null,"message":"Rate limit exceeded","param":null,"type":"rate_limit_exceeded"}}\n', 'code': 429} After invoking runnable_with_retries
- Assistant
It seems that the retry mechanism did not function as expected, resulting in the exception being thrown on the first attempt without retrying. Let's go through a few possible reasons and their solutions:
-
Ensure the Retry Configuration is Set Correctly:
- Make sure that the
runnable_with_retriesis correctly configured to handle exceptions and retry upon encountering them. - Double-check that the retry mechanism is being correctly applied to your chain and runnable.
- Make sure that the
-
Verify LLM Mocking and Integration:
- Ensure that all components of the original function, including any local variables or setup like the llm object, are correctly mocked.
- If
ChatOpenAIis used inRunnableLambda, ensure it's correctly passed or replaced with a mocked version.
-
Log the Number of Attempts:
- Consider adding print statements or logs inside the mock to track how many times the mocked function
mock_invokeis being executed. This can assist in verifying if the retry attempts are processed correctly.
- Consider adding print statements or logs inside the mock to track how many times the mocked function
-
Check the Patch Target:
- Verify that the
patchtarget is correct. The string insidepatchshould match the actual location of theinvokemethod being called in your application code path.
- Verify that the
Here's an adjusted approach to verify retry attempts and correct usage of the retry mechanism:
import unittest from unittest.mock import AsyncMock, patch from langchain_core.prompts import PromptTemplate from langchain_openai import ChatOpenAI import asyncio attempts = 0 # Track the number of retry attempts # Define the mock function for simulating errors followed by a success async def mock_invoke(*args, **kwargs): global attempts attempts += 1 if attempts < 3: # Simulate error for first two calls raise ValueError({'message': '{"error":{"code":null,"message":"Rate limit exceeded","param":null,"type":"rate_limit_exceeded"}}\n', 'code': 429}) return {"text": "Why do programmers prefer dark mode? Because light attracts bugs!"} # Method 1 async def test_method_1(): global attempts attempts = 0 with patch('langchain_openai.ChatOpenAI.invoke', new_callable=AsyncMock) as invoke_mock: invoke_mock.side_effect = mock_invoke # Define prompt and model template = PromptTemplate.from_template("Tell me a joke about {topic}.") # Instantiate ChatOpenAI as necessary in this context llm = ChatOpenAI() # Add retry logic to the model invocation runnable_with_retries = llm.with_retry( retry_if_exception_type=(ValueError, ), wait_exponential_jitter=False, # False or modify as needed for testing stop_after_attempt=3 ) # Chain with template and retry-enabled model chain = template | runnable_with_retries # Invoke and verify retry print("Before invoking chain") try: result = await chain.invoke({"topic": "programmers"}) print(f"Method 1: Received result: {result}") except ValueError as e: print(f"Method 1: Unexpected exception: {e}") print("After invoking chain") print(f"Retries attempted: {attempts}") # Method 2 async def test_method_2(): global attempts attempts = 0 with patch('langchain_openai.ChatOpenAI.invoke', new_callable=AsyncMock) as invoke_mock: invoke_mock.side_effect = mock_invoke # Instantiate ChatOpenAI as necessary in this context llm = ChatOpenAI() # Wrap the model invocation in a Runnable runnable = RunnableLambda(lambda input: llm.invoke(input)) # Configure retries runnable_with_retries = RunnableRetry( bound=runnable, retry_exception_types=(ValueError, ), max_attempt_number=3, wait_exponential_jitter=False ) # Invoke and verify retry print("Before invoking runnable_with_retries") try: result = await runnable_with_retries.invoke("Tell me a joke.") print(f"Method 2: Received result: {result}") except ValueError as e: print(f"Method 2: Unexpected exception: {e}") print("After invoking runnable_with_retries") print(f"Retries attempted: {attempts}") # Run the tests await test_method_1() await test_method_2()Explanation:
- Global
attemptsCounter: A global variableattemptsis used to track the number of attempts made to invoke the mock method. mock_invokeLogic: Themock_invokefunction raises aValueErrorfor the first two calls and returns a success on the third.- Patch Verification and Initialization: Ensuring
ChatOpenAI()is mocked or instantiated correctly and the retry logic is correctly applied. - Disable Jitter for Testing: Temporarily setting
wait_exponential_jitter=Falseallows for easier testing and verification without delay.
If the retry mechanism isn't being applied, you might need to check elements like the exact integration of
with_retryand make sure your patch target withinpatch()accurately refers to what you're testing. -
- User
this was the output:
Before invoking chain Method 1: Unexpected exception: {'message': '{"error":{"code":null,"message":"Rate limit exceeded","param":null,"type":"rate_limit_exceeded"}}\n', 'code': 429} After invoking chain Retries attempted: 1 Before invoking runnable_with_retries Method 2: Unexpected exception: {'message': '{"error":{"code":null,"message":"Rate limit exceeded","param":null,"type":"rate_limit_exceeded"}}\n', 'code': 429} After invoking runnable_with_retries Retries attempted: 1
- Assistant
It seems that the retry logic might not be working as expected, leading to the
ValueErrorbeing raised on the first attempt without retrying. Here are some steps to diagnose and fix the issue:-
Check Retry Logic Setup:
- Ensure that both methods (
llm.with_retryandRunnableRetry) are set up to handleValueErrorexceptions specifically. If these configurations are not correctly applying the retry logic, it could cause the issue you're facing.
- Ensure that both methods (
-
Verify Object Instantiation:
- Ensure the model
ChatOpenAIandPromptTemplateare correctly instantiated. Ifwith_retryorRunnableRetryare not designed to handle the mocking structure, it may not work as intended.
- Ensure the model
-
Fix Mocking Scope:
- Ensure the patch target is correctly pointing to where
invokeis used in your context. - Check whether any wrappers or adapters in
with_retryorRunnableRetrymight intercede, potentially affecting the retry attempts.
- Ensure the patch target is correctly pointing to where
-
Debugging Steps:
- Add logging in the
mock_invokeandrunnable_with_retriesto trace execution flows. - Check if any try-except blocks around retry mechanisms are inadvertently catching the exceptions, preventing retry logic from kicking in.
- Add logging in the
-
Direct Retry Configuration Test:
- Simplify the test by isolating logic to explicitly confirm that
with_retryandRunnableRetrycan perform retries as expected when directly instantiated.
- Simplify the test by isolating logic to explicitly confirm that
Below is a revised breakdown which ensures simplicity in testing these constructs:
import unittest from unittest.mock import AsyncMock, patch from langchain_core.prompts import PromptTemplate from langchain_openai import ChatOpenAI from langchain_core.runnables import RunnableLambda from langchain_core.runnables.retry import RunnableRetry import asyncio attempts = 0 # Track the number of retry attempts # Define the mock function to simulate two failures followed by success async def mock_invoke(*args, **kwargs): global attempts attempts += 1 if attempts <= 2: # Trigger error for the first two invocations raise ValueError({ 'message': '{"error":{"code":null,"message":"Rate limit exceeded","param":null,"type":"rate_limit_exceeded"}}\n', 'code': 429 }) return {"text": "Why do programmers prefer dark mode? Because light attracts bugs!"} # Method 1 async def test_method_1(): global attempts attempts = 0 # Reset attempt counter at start of test with patch('langchain_openai.ChatOpenAI.invoke', new_callable=AsyncMock) as invoke_mock: invoke_mock.side_effect = mock_invoke # Define prompt and model template = PromptTemplate.from_template("Tell me a joke about {topic}.") # Direct attempt at using retries with ChatOpenAI llm = ChatOpenAI() runnable_with_retries = llm.with_retry( retry_if_exception_type=(ValueError, ), wait_exponential_jitter=False, # Disable jitter to quickly test retries stop_after_attempt=3 ) # Create a chain with the template and the retry-enabled model chain = template | runnable_with_retries print("Before invoking chain") try: result = await chain.invoke({"topic": "programmers"}) print(f"Method 1: Received result: {result}") except Exception as e: print(f"Method 1: Unexpected exception: {e}") print("After invoking chain") print(f"Retries attempted for Method 1: {attempts}") # Method 2 async def test_method_2(): global attempts attempts = 0 # Reset attempt counter at start of test with patch('langchain_openai.ChatOpenAI.invoke', new_callable=AsyncMock) as invoke_mock: invoke_mock.side_effect = mock_invoke llm = ChatOpenAI() # Wrap the model invocation in a Runnable runnable = RunnableLambda(lambda input: llm.invoke(input)) # Configure retries runnable_with_retries = RunnableRetry( bound=runnable, retry_exception_types=(ValueError, ), max_attempt_number=3, wait_exponential_jitter=False # Simplified for testing purposes ) print("Before invoking runnable_with_retries") try: result = await runnable_with_retries.invoke("Tell me a joke.") print(f"Method 2: Received result: {result}") except Exception as e: print(f"Method 2: Unexpected exception: {e}") print("After invoking runnable_with_retries") print(f"Retries attempted for Method 2: {attempts}") # Run the tests asyncio.run(test_method_1()) asyncio.run(test_method_2())Key Adjustments:
- Ensure Mock Setup Properly Controls Call Conditions: The mock is set to simulate two failed attempts followed by success, directly replaying the scenario of retry.
- Logging Attempts: I added logging indicators to monitor retry attempts specifically, verifying if the retry mechanism is correctly incrementing.
- Direct Testing of Callable Retry Mechanism: This scenario tests your retry classes in isolation to ensure they're executing.
This setup should reliably test whether retries are being executed through either of the retry policies (
with_retryorRunnableRetry). -
- User
it is not working but now I can understand why,if we change the invoke method, it won't call the original one, maybe this is interfering with how we set up the code
- Assistant
Given your understanding that the retry logic might not be correctly configured due to how the mocking interferes with the original method call, let's take a deep dive into how to properly integrate mocking with retry logic.
Here's a detailed plan to help isolate the retry logic for successful integration with mock tests:
-
Ensure Mocking Targets the Correct Method: This involves ensuring that you correctly target the function where the
invokemethod is actually called in your workflow or codebase, i.e., whereinvokeis originally utilized inChatOpenAI. -
Recreate Retry Configuration if Needed: If retry logic is custom or otherwise dependent on specific handling, ensure that your mock retains the required callable chain correctly through retry attempts.
-
Use Internal Mocking Methods: Instead of complex setups, use Python's inherent mocking capabilities to control retries.
Here’s a focused example demonstrating how one might correctly configure and test retry logic using mocking:
import unittest from unittest.mock import AsyncMock, patch from langchain_core.prompts import PromptTemplate from langchain_openai import ChatOpenAI from langchain_core.runnables import RunnableLambda from langchain_core.runnables.retry import RunnableRetry import asyncio # Global attempt counters attempts_method_1 = 0 attempts_method_2 = 0 # Mock invoke function with retry logic incorporated async def mock_successful_invoke(*args, **kwargs): return {"text": "Why do programmers prefer dark mode? Because light attracts bugs!"} async def mock_failing_invoke(*args, **kwargs): raise ValueError({ 'message': '{"error":{"code":null,"message":"Rate limit exceeded","param":null,"type":"rate_limit_exceeded"}}\n', 'code': 429 }) # Testing Method 1 async def test_method_1(): global attempts_method_1 attempts_method_1 = 0 with patch.object(ChatOpenAI, 'invoke', new_callable=AsyncMock) as invoke_mock: # Setup mock side effects invoke_mock.side_effect = [mock_failing_invoke(), mock_failing_invoke(), mock_successful_invoke()] template = PromptTemplate.from_template("Tell me a joke about {topic}.") llm = ChatOpenAI() # Create retry-enabled invoke runnable_with_retries = llm.with_retry( retry_if_exception_type=(ValueError, ), wait_exponential_jitter=False, stop_after_attempt=3 ) chain = template | runnable_with_retries print("Before invoking chain") try: result = await chain.invoke({"topic": "programmers"}) print(f"Method 1: Received result: {result}") except ValueError as e: print(f"Method 1: Caught exception: {e}") print("After invoking chain") print(f"Retries attempted for Method 1: {invoke_mock.call_count}") # Testing Method 2 async def test_method_2(): global attempts_method_2 attempts_method_2 = 0 with patch.object(ChatOpenAI, 'invoke', new_callable=AsyncMock) as invoke_mock: # Setup mock side effects invoke_mock.side_effect = [mock_failing_invoke(), mock_failing_invoke(), mock_successful_invoke()] llm = ChatOpenAI() runnable = RunnableLambda(lambda input: llm.invoke(input)) runnable_with_retries = RunnableRetry( bound=runnable, retry_exception_types=(ValueError, ), max_attempt_number=3, wait_exponential_jitter=False ) print("Before invoking runnable_with_retries") try: result = await runnable_with_retries.invoke("Tell me a joke.") print(f"Method 2: Received result: {result}") except ValueError as e: print(f"Method 2: Caught exception: {e}") print("After invoking runnable_with_retries") print(f"Retries attempted for Method 2: {invoke_mock.call_count}") # Run the tests asyncio.run(test_method_1()) asyncio.run(test_method_2())Key Changes:
-
Mock Object Configuration using Object Method: Uses
patch.objecttargeting theChatOpenAIclass to more reliably interceptinvokemethod calls, ensuring full control over its behavior. -
Defined
side_effectList: Clearly definedside_effectlist with successive failures followed by success allows easy tracing of how the retry logic reacts. -
Relying on Mock's
call_count: Instead of manual attempts counting, leveraginginvoke_mock.call_countprovides accurate insight into how many attempts were made, helping diagnose any discrepancies in retry handling.
This setup streamlines the testing of retry logic by reflecting a typical sequence involving repeated request attempts, where failovers gracefully lead to a successful outcome after exhaustion of prior retry logic.
-
- User
it gave me this output:
Before invoking chain Method 1: Received result: <coroutine object mock_failing_invoke at 0x00000231B15B49E0> After invoking chain Retries attempted for Method 1: 1 Before invoking runnable_with_retries Method 2: Received result: <coroutine object mock_failing_invoke at 0x00000231B04DA340> After invoking runnable_with_retries Retries attempted for Method 2: 1
- Assistant
The issue here appears to be that the
side_effectlist contains coroutine objects rather than their execution results. When you're using coroutines with side effects in Python'sunittest.mock, it's crucial to ensure that the actual calls await these coroutine functions. Let's adjust the implementation to correctly handle the asynchronous behavior and evaluate the execution ofmock_failing_invokeandmock_successful_invoke.Here's how you can fix this:
-
Properly Await Coroutines in the Side Effect: To ensure the coroutines are awaited during testing, you'll need to define them in a way that fits into the side effect call chain or await them as they are executed.
-
Adjust the
side_effectList Entries: Instead of placing coroutines directly in the list, wrap them so they execute correctly upon being called.
Here's an updated example, fixing this behavior:
import unittest from unittest.mock import AsyncMock, patch from langchain_core.prompts import PromptTemplate from langchain_openai import ChatOpenAI from langchain_core.runnables import RunnableLambda from langchain_core.runnables.retry import RunnableRetry import asyncio # Mock invoke function for simulating async responses async def mock_failing_invoke(*args, **kwargs): raise ValueError({ 'message': '{"error":{"code":null,"message":"Rate limit exceeded","param":null,"type":"rate_limit_exceeded"}}\n', 'code': 429 }) async def mock_successful_invoke(*args, **kwargs): return {"text": "Why do programmers prefer dark mode? Because light attracts bugs!"} # Testing Method 1 async def test_method_1(): with patch.object(ChatOpenAI, 'invoke', new_callable=AsyncMock) as invoke_mock: # Setup mock to call the coroutine functions invoke_mock.side_effect = [ mock_failing_invoke, # Call the function itself, which will be awaited mock_failing_invoke, mock_successful_invoke ] template = PromptTemplate.from_template("Tell me a joke about {topic}.") llm = ChatOpenAI() # Create retry-enabled invoke runnable_with_retries = llm.with_retry( retry_if_exception_type=(ValueError, ), wait_exponential_jitter=False, stop_after_attempt=3 ) chain = template | runnable_with_retries print("Before invoking chain") try: result = await chain.invoke({"topic": "programmers"}) print(f"Method 1: Received result: {result}") except ValueError as e: print(f"Method 1: Caught expected exception: {e}") print("After invoking chain") print(f"Retries attempted for Method 1: {invoke_mock.call_count}") # Testing Method 2 async def test_method_2(): with patch.object(ChatOpenAI, 'invoke', new_callable=AsyncMock) as invoke_mock: # Setup mock to call the coroutine functions invoke_mock.side_effect = [ mock_failing_invoke, mock_failing_invoke, mock_successful_invoke ] llm = ChatOpenAI() runnable = RunnableLambda(lambda input: llm.invoke(input)) runnable_with_retries = RunnableRetry( bound=runnable, retry_exception_types=(ValueError, ), max_attempt_number=3, wait_exponential_jitter=False ) print("Before invoking runnable_with_retries") try: result = await runnable_with_retries.invoke("Tell me a joke.") print(f"Method 2: Received result: {result}") except ValueError as e: print(f"Method 2: Caught expected exception: {e}") print("After invoking runnable_with_retries") print(f"Retries attempted for Method 2: {invoke_mock.call_count}") # Run the tests asyncio.run(test_method_1()) asyncio.run(test_method_2())Key Adjustments:
-
side_effectEntries: The functions inside_effectare specified as functions (not coroutine objects) whichinvoke_mockwill call and properly await during the mock invocation. -
Awaiting Outcomes: When
invokeis called, the mock will execute these functions, ensuring they are awaited correctly as part of the invocation process.
This approach ensures asynchronous actions are correctly handled, counting retries correctly across the different stages of a mocked function call.
-
- User
same output:
Before invoking chain Method 1: Received result: <function mock_failing_invoke at 0x00000231B12772E0> After invoking chain Retries attempted for Method 1: 1 Before invoking runnable_with_retries Method 2: Received result: <function mock_failing_invoke at 0x00000231B12772E0> After invoking runnable_with_retries Retries attempted for Method 2: 1
- Assistant
It seems like there was a misunderstanding in setting up the side effects to correctly simulate a sequence of failures followed by a success. The side effects need to be coroutines when you're using async mocks, which means you should be returning the actual coroutine for each step in the side effect sequence. When you define a mock side effect in an async context, each entry should be an awaitable function.
Let's correct this by using lambda functions to return the coroutine as part of the side effect list:
import unittest from unittest.mock import AsyncMock, patch from langchain_core.prompts import PromptTemplate from langchain_openai import ChatOpenAI from langchain_core.runnables import RunnableLambda from langchain_core.runnables.retry import RunnableRetry import asyncio async def mock_failing_invoke(*args, **kwargs): raise ValueError({ 'message': '{"error":{"code":null,"message":"Rate limit exceeded","param":null,"type":"rate_limit_exceeded"}}\n', 'code': 429 }) async def mock_successful_invoke(*args, **kwargs): return {"text": "Why do programmers prefer dark mode? Because light attracts bugs!"} # Testing Method 1 async def test_method_1(): with patch.object(ChatOpenAI, 'invoke', new_callable=AsyncMock) as invoke_mock: # Setup mock behavior with lambda to correctly await coroutines invoke_mock.side_effect = [ lambda *args, **kwargs: mock_failing_invoke(*args, **kwargs), lambda *args, **kwargs: mock_failing_invoke(*args, **kwargs), lambda *args, **kwargs: mock_successful_invoke(*args, **kwargs) ] template = PromptTemplate.from_template("Tell me a joke about {topic}.") llm = ChatOpenAI() # Create retry-enabled invoke runnable_with_retries = llm.with_retry( retry_if_exception_type=(ValueError, ), wait_exponential_jitter=False, stop_after_attempt=3 ) chain = template | runnable_with_retries print("Before invoking chain") try: result = await chain.invoke({"topic": "programmers"}) print(f"Method 1: Received result: {result}") except ValueError as e: print(f"Method 1: Caught expected exception: {e}") print("After invoking chain") print(f"Retries attempted for Method 1: {invoke_mock.call_count}") # Testing Method 2 async def test_method_2(): with patch.object(ChatOpenAI, 'invoke', new_callable=AsyncMock) as invoke_mock: # Setup mock behavior with lambda to correctly await coroutines invoke_mock.side_effect = [ lambda *args, **kwargs: mock_failing_invoke(*args, **kwargs), lambda *args, **kwargs: mock_failing_invoke(*args, **kwargs), lambda *args, **kwargs: mock_successful_invoke(*args, **kwargs) ] llm = ChatOpenAI() runnable = RunnableLambda(lambda input: llm.invoke(input)) runnable_with_retries = RunnableRetry( bound=runnable, retry_exception_types=(ValueError, ), max_attempt_number=3, wait_exponential_jitter=False ) print("Before invoking runnable_with_retries") try: result = await runnable_with_retries.invoke("Tell me a joke.") print(f"Method 2: Received result: {result}") except ValueError as e: print(f"Method 2: Caught expected exception: {e}") print("After invoking runnable_with_retries") print(f"Retries attempted for Method 2: {invoke_mock.call_count}") # Run the tests asyncio.run(test_method_1()) asyncio.run(test_method_2())Explanation of Changes:
-
Lambda Functions: Wrapped each coroutine call within a lambda function. This ensures that when the
invoke_mockuses these side effects, it correctly invokes each coroutine. By defining lambdas, the function is not executed immediately but is instead returned to be called later, allowing theasyncnature to be preserved. -
Correctly Set Up Async Await: This approach ensures the functions being run are actually awaited, simulating the retry mechanism by counting attempts correctly and proceeding through the side effects as intended.
This should help verify that your retry mechanism correctly attempts each call.
-
- User
output:
Before invoking chain Method 1: Received result: <function test_method_1.. at 0x00000231B1277D80> After invoking chain Retries attempted for Method 1: 1 Before invoking runnable_with_retries Method 2: Received result: <function test_method_2.. at 0x00000231B1277880> After invoking runnable_with_retries Retries attempted for Method 2: 1
So it didn't changed much, maybe the problem is in the original llm object
- Assistant
It seems that the retry mechanism isn't functioning as expected, and the issue might be related to the retry configuration itself or how the LangChain objects are handling retries. Let's take a step-by-step approach to ensure that everything is correctly configured.
Key Steps
-
Check Retry Configuration: Ensure your
with_retrymethod andRunnableRetryobjects are correctly configured to handle retries. Ensure they're correctly connected to the parts of the code where exceptions might occur. -
Mock Correct Object/Method: Verify the exact path of the method being mocked with
patch. Ensure it reflects how and where theinvokemethod is being used in the LangChain's scope. -
Log Actual Calls: You can log what methods get called, effectively serving as a debug to ensure what's being hit during retries.
-
Isolate Retry Logic: Simplify the test or simulate a retry function to ascertain whether the retry logic itself behaves correctly.
-
Review Object Construction: Given that
llm(likelyChatOpenAI) is throughout the LangChain ecosystem, confirm that the instantiation (ChatOpenAI()) correctly points to proper configurations that are intercepted by your mock.
Below is a revised example that sets the lambda side effects directly, ensuring execution relevance for each step in asynchronous execution:
import unittest from unittest.mock import AsyncMock, patch from langchain_core.prompts import PromptTemplate from langchain_openai import ChatOpenAI from langchain_core.runnables import RunnableLambda from langchain_core.runnables.retry import RunnableRetry import asyncio # Mock invoke function simulating async responses async def mock_failing_invoke(*args, **kwargs): raise ValueError({ 'message': '{"error":{"code":null,"message":"Rate limit exceeded","param":null,"type":"rate_limit_exceeded"}}\n', 'code': 429 }) async def mock_successful_invoke(*args, **kwargs): return {"text": "Why do programmers prefer dark mode? Because light attracts bugs!"} # Testing Method 1 async def test_method_1(): with patch.object(ChatOpenAI, 'invoke', new_callable=AsyncMock) as invoke_mock: # Setup mock behavior with awaiting lambdas invoke_mock.side_effect = [ lambda *args, **kwargs: mock_failing_invoke(), lambda *args, **kwargs: mock_failing_invoke(), lambda *args, **kwargs: mock_successful_invoke() ] template = PromptTemplate.from_template("Tell me a joke about {topic}.") llm = ChatOpenAI() # Create retry-enabled invoke runnable_with_retries = llm.with_retry( retry_if_exception_type=(ValueError, ), wait_exponential_jitter=False, stop_after_attempt=3 ) chain = template | runnable_with_retries print("Before invoking chain") try: result = await chain.invoke({"topic": "programmers"}) print(f"Method 1: Received result: {result}") except ValueError as e: print(f"Method 1: Caught expected exception: {e}") print("After invoking chain") print(f"Retries attempted for Method 1: {invoke_mock.call_count}") # Testing Method 2 async def test_method_2(): with patch.object(ChatOpenAI, 'invoke', new_callable=AsyncMock) as invoke_mock: # Setup mock behavior with awaiting lambdas invoke_mock.side_effect = [ lambda *args, **kwargs: mock_failing_invoke(), lambda *args, **kwargs: mock_failing_invoke(), lambda *args, **kwargs: mock_successful_invoke() ] llm = ChatOpenAI() runnable = RunnableLambda(lambda input: llm.invoke(input)) runnable_with_retries = RunnableRetry( bound=runnable, retry_exception_types=(ValueError,), max_attempt_number=3, wait_exponential_jitter=False ) print("Before invoking runnable_with_retries") try: result = await runnable_with_retries.invoke("Tell me a joke.") print(f"Method 2: Received result: {result}") except ValueError as e: print(f"Method 2: Caught expected exception: {e}") print("After invoking runnable_with_retries") print(f"Retries attempted for Method 2: {invoke_mock.call_count}") # Run the tests asyncio.run(test_method_1()) asyncio.run(test_method_2())Points of Consideration:
- Ensure Valid Path: Verify that
patch.objectis pointing at the correct class and method ofinvoke. - Consistent Lambda Use: Use lambda functions that correctly call awaited coroutine functions without ambiguity.
- Validate
llmConfiguration: Double-checkrunnable_with_retriesand similar elements to ensure they enact retry logic. - Logging Internals: Add debug outputs within retry sections to trace what's being awaited and handled.
This approach aims to ensure testing covers a full retry cycle robustly within the LangChain framework.
-