Skip to content
TheChatArchiveThe Chat Archive
Claudeclaude-3-7-sonnet-20250219-thinking-32k· AnthropicHugging Face LMArena Human Preference 140K

I’m using boto3 to upload file objects to an S3 compatible storage. I’m using…

From a Chatbot Arena battleThe same person sent this prompt to two anonymous models at once. Compare it with Qwenqwen3-235b-a22b’s answer.
  1. User

    I’m using boto3 to upload file objects to an S3 compatible storage. I’m using send_fileobj to upload large directories compressed as tar archives. I’m getting HTTP 500 errors. How to retry these errors? Won’t my file cursor position prevent retries from being correct? What if I wanted to retry HTTP 400 as well?

  2. Claude Assistant

    To handle HTTP 500 errors when uploading large files to S3 with retry support, you'll need to address both the retry configuration and the file cursor position issue. Here's how:

    import boto3
    from botocore.config import Config
    from botocore.exceptions import ClientError
    import io
    import os
    
    # Create a rewindable file wrapper
    class RewindableFileWrapper:
        def __init__(self, file_obj):
            self.file_obj = file_obj
            self.initial_position = file_obj.tell()
        
        def seek(self, position, whence=0):
            return self.file_obj.seek(position, whence)
        
        def tell(self):
            return self.file_obj.tell()
        
        def read(self, size=-1):
            return self.file_obj.read(size)
        
        def rewind(self):
            self.file_obj.seek(self.initial_position)
    
    # Configure retry settings
    retry_config = Config(
        retries={
            'max_attempts': 5,
            'mode': 'adaptive',  # Automatically adjusts backoff based on error
        }
    )
    
    # Create S3 client with custom retry config
    s3_client = boto3.client('s3', config=retry_config)
    
    def upload_with_retry(file_path, bucket, key, include_400_retry=False):
        with open(file_path, 'rb') as file_obj:
            rewindable_file = RewindableFileWrapper(file_obj)
            
            max_attempts = 5
            attempt = 0
            
            while attempt < max_attempts:
                try:
                    s3_client.upload_fileobj(rewindable_file, bucket, key)
                    print(f"Successfully uploaded {key}")
                    return True
                except ClientError as e:
                    error_code = e.response.get('ResponseMetadata', {}).get('HTTPStatusCode')
                    
                    # Handle HTTP 500 errors
                    if error_code >= 500:
                        attempt += 1
                        if attempt < max_attempts:
                            print(f"Got 5xx error ({error_code}). Retrying {attempt}/{max_attempts}...")
                            rewindable_file.rewind()
                        else:
                            print(f"Failed after {max_attempts} attempts")
                            raise
                    # Handle HTTP 400 errors if requested
                    elif include_400_retry and error_code >= 400 and error_code < 500:
                        attempt += 1
                        if attempt < max_attempts:
                            print(f"Got 4xx error ({error_code}). Retrying {attempt}/{max_attempts}...")
                            rewindable_file.rewind()
                        else:
                            print(f"Failed after {max_attempts} attempts")
                            raise
                    else:
                        # Don't retry other errors
                        raise
    
    # Usage
    upload_with_retry('my_large_archive.tar.gz', 'my-bucket', 'archives/my_large_archive.tar.gz', include_400_retry=True)
    

    Would you like me to explain or break down this code?

Responses were produced in different, uncontrolled contexts (system prompts, settings, prior turns). This is not a benchmark.