Skip to content
TheChatArchiveThe Chat Archive
o1-mini-2024-09-12· OpenAIAi2 WildChat-4.8M (AI2)

use std::fs; use std::path::{Path, PathBuf}; use std::time::Duration; use…

  1. User

    use std::fs; use std::path::{Path, PathBuf}; use std::time::Duration; use tokio::fs::{create_dir_all, File}; use tokio::io::AsyncWriteExt; use reqwest::Client; use futures::stream::{self, StreamExt}; use tracing::{error, info};

    #[tokio::main] async fn main() { tracing_subscriber::fmt::init(); // Base directory where the downloads will be saved let base_dir = Path::new(r"C:\Users\Lukas\Downloads\lkjhgfd2");

    // List of placeIDs
    let place_ids = vec![
        "10789565990",
        "11181971058",
        "11181982436",
        "11182034319",
        "11182037286",
        "11263735303",
        "11263873956",
        "12647442701",
        "33564694",
        "4568576845",
        "4737314314",
        "4748963513",
        "4956810680",
        "4998492434",
        "5000291183",
        "5063342777",
        "5229799451",
        "5400427674",
        "5465723678",
        "5914914682",
        "5972110179",
        "6087359275",
        "6701464352",
        "6767882272",
        "6768240609",
        "6783491168",
        "7162161171",
        "7730658758",
        "9048939285",
    ];
    
    // Initialize HTTP client with timeout and other configurations
    let client = Client::builder()
        .timeout(Duration::from_secs(30))
        .build()
        .expect("Failed to build HTTP client");
    
    // Process each placeID concurrently with a limit on concurrency
    let concurrency_limit = 5; // Adjust based on your network capability
    stream::iter(place_ids)
        .for_each_concurrent(concurrency_limit, |place_id| {
            let client = client.clone();
            let base_dir = base_dir.to_path_buf();
            async move {
                if let Err(e) = process_place_id(&client, &base_dir, place_id).await {
                    error!("Error processing placeID {}: {}", place_id, e);
                }
            }
        })
        .await;
    
    info!("All downloads completed.");
    

    }

    /// Processes a single placeID: /// - Reads index.txt /// - Parses entries /// - Downloads assets async fn process_place_id(client: &Client, base_dir: &PathBuf, place_id: &str) -> Result<(), Box> { info!("Processing placeID: {}", place_id);

    // Path to index.txt
    let index_path = base_dir.join(place_id).join("index.txt");
    
    // Read index.txt content
    let index_content = match fs::read_to_string(&index_path) {
        Ok(content) => content,
        Err(e) => {
            error!("Failed to read {}: {}", index_path.display(), e);
            return Err(Box::new(e));
        }
    };
    
    // Parse index.txt lines
    let assets: Vec<(String, String)> = index_content
        .lines()
        .filter_map(|line| {
            let parts: Vec<&str> = line.split('|').map(|s| s.trim()).collect();
            if parts.len() == 2 {
                Some((parts[0].to_string(), parts[1].to_string()))
            } else {
                None
            }
        })
        .collect();
    info!("---------------------");
    info!("Found {} assets for placeID {}", assets.len(), place_id);
    info!("---------------------");
    
    // Define concurrency for asset downloads per placeID
    let asset_concurrency = 10; // Adjust as needed
    
    // Process asset downloads concurrently
    stream::iter(assets)
        .for_each_concurrent(asset_concurrency, |(version, url)| {
            let client = client.clone();
            let base_dir = base_dir.clone();
            let place_id = place_id.to_string();
            async move {
                if let Err(e) = download_asset(&client, &base_dir, &place_id, &version, &url).await {
                    error!("Failed to download asset {} for placeID {}: {}", url, place_id, e);
                }
            }
        })
        .await;
    
    info!("Completed placeID: {}", place_id);
    Ok(())
    

    }

    /// Downloads a single asset and saves it to the designated path async fn download_asset( client: &Client, base_dir: &PathBuf, place_id: &str, version: &str, url: &str, ) -> Result<(), Box> { // Extract asset filename from URL let original_filename = match url.split('/').last() { Some(name) => name, None => { return Err("Failed to extract asset filename from URL".into()); } };

    // Append ".rbxl" extension
    let asset_filename = format!("{}.rbxl", original_filename);
    
    // Define the target directory and file path
    let target_dir = base_dir.join(place_id).join(version);
    let target_path = target_dir.join(&asset_filename);
    
    // Create directories if they don't exist
    create_dir_all(&target_dir).await?;
    
    // Check if the file already exists to avoid re-downloading
    if target_path.exists() {
        info!("Asset already exists: {:?}", target_path);
        return Ok(());
    }
    
    // Make the HTTP GET request
    let response = client.get(url).send().await?;
    
    if !response.status().is_success() {
        return Err(format!("Failed to download {}: HTTP {}", url, response.status()).into());
    }
    
    // Stream the response bytes to the file
    let mut file = File::create(&target_path).await?;
    let mut stream = response.bytes_stream();
    
    while let Some(chunk) = stream.next().await {
        let data = chunk?;
        file.write_all(&data).await?;
    }
    
    info!("Downloaded: {:?}", target_path);
    Ok(())
    

    }

    Make it so it downloads somewhere else than the path that has the index.txts and stuff.

    Make it so it doesn't create files like C:\Users\Lukas\Downloads\lkjhgfd2\11263735303\1{hash}.rbxl More like C:\Users\Lukas\Downloads\lkjhgfd2\11263735303\1_{hash}.rbxl

    e.x if my path for reading the index.txts was C:\Users\Lukas\Downloads\lkjhgfd2

    my path for the downloads urls would be like C:\Users\Lukas\Downloads\lkjhgfd3\11263735303\1_{hash}.rbxl to remove confusion

  2. Assistant

    Sure! Let's modify your Rust program to meet the following requirements:

    1. Separate Directories for Reading and Downloading:

      • Reading Directory (e.g., lkjhgfd2): This is where your index.txt files are located.
      • Download Directory (e.g., lkjhgfd3): This is where the downloaded assets will be saved.
    2. Custom File Naming Convention:

      • Instead of creating separate subdirectories for each version, the files will be named using the format {version}_{hash}.rbxl and saved directly within the download_dir/place_id directory.

    Updated Code

    Below is the updated Rust code with the necessary modifications and comments explaining the changes:

    use std::fs;
    use std::path::{Path, PathBuf};
    use std::time::Duration;
    use tokio::fs::{create_dir_all, File};
    use tokio::io::AsyncWriteExt;
    use reqwest::Client;
    use futures::stream::{self, StreamExt};
    use tracing::{error, info};
    
    #[tokio::main]
    async fn main() {
        // Initialize the tracing subscriber for logging
        tracing_subscriber::fmt::init();
    
        // Define the base directory where index.txt files are located (reading directory)
        let read_dir = Path::new(r"C:\Users\Lukas\Downloads\lkjhgfd2");
    
        // Define the base directory where downloads will be saved (download directory)
        let download_dir = Path::new(r"C:\Users\Lukas\Downloads\lkjhgfd3");
    
        // Ensure the download directory exists
        if let Err(e) = fs::create_dir_all(&download_dir) {
            error!(
                "Failed to create download directory {}: {}",
                download_dir.display(),
                e
            );
            return;
        }
    
        // List of placeIDs
        let place_ids = vec![
            "10789565990",
            "11181971058",
            "11181982436",
            "11182034319",
            "11182037286",
            "11263735303",
            "11263873956",
            "12647442701",
            "33564694",
            "4568576845",
            "4737314314",
            "4748963513",
            "4956810680",
            "4998492434",
            "5000291183",
            "5063342777",
            "5229799451",
            "5400427674",
            "5465723678",
            "5914914682",
            "5972110179",
            "6087359275",
            "6701464352",
            "6767882272",
            "6768240609",
            "6783491168",
            "7162161171",
            "7730658758",
            "9048939285",
        ];
    
        // Initialize HTTP client with timeout and other configurations
        let client = Client::builder()
            .timeout(Duration::from_secs(30))
            .build()
            .expect("Failed to build HTTP client");
    
        // Process each placeID concurrently with a limit on concurrency
        let concurrency_limit = 5; // Adjust based on your network capability
        stream::iter(place_ids)
            .for_each_concurrent(concurrency_limit, |place_id| {
                let client = client.clone();
                let read_dir = read_dir.to_path_buf();
                let download_dir = download_dir.to_path_buf();
                async move {
                    if let Err(e) = process_place_id(&client, &read_dir, &download_dir, place_id).await {
                        error!("Error processing placeID {}: {}", place_id, e);
                    }
                }
            })
            .await;
    
        info!("All downloads completed.");
    }
    
    /// Processes a single placeID:
    /// - Reads index.txt from the reading directory
    /// - Parses entries
    /// - Downloads assets to the download directory
    async fn process_place_id(
        client: &Client,
        read_dir: &PathBuf,
        download_dir: &PathBuf,
        place_id: &str,
    ) -> Result<(), Box<dyn std::error::Error>> {
        info!("Processing placeID: {}", place_id);
    
        // Path to index.txt in the reading directory
        let index_path = read_dir.join(place_id).join("index.txt");
    
        // Read index.txt content
        let index_content = match fs::read_to_string(&index_path) {
            Ok(content) => content,
            Err(e) => {
                error!("Failed to read {}: {}", index_path.display(), e);
                return Err(Box::new(e));
            }
        };
    
        // Parse index.txt lines into a vector of (version, url) tuples
        let assets: Vec<(String, String)> = index_content
            .lines()
            .filter_map(|line| {
                let parts: Vec<&str> = line.split('|').map(|s| s.trim()).collect();
                if parts.len() == 2 {
                    Some((parts[0].to_string(), parts[1].to_string()))
                } else {
                    None
                }
            })
            .collect();
    
        info!("---------------------");
        info!("Found {} assets for placeID {}", assets.len(), place_id);
        info!("---------------------");
    
        // Define concurrency for asset downloads per placeID
        let asset_concurrency = 10; // Adjust as needed
    
        // Process asset downloads concurrently
        stream::iter(assets)
            .for_each_concurrent(asset_concurrency, |(version, url)| {
                let client = client.clone();
                let download_dir = download_dir.clone();
                let place_id = place_id.to_string();
                async move {
                    if let Err(e) = download_asset(&client, &download_dir, &place_id, &version, &url).await {
                        error!(
                            "Failed to download asset {} for placeID {}: {}",
                            url, place_id, e
                        );
                    }
                }
            })
            .await;
    
        info!("Completed placeID: {}", place_id);
        Ok(())
    }
    
    /// Downloads a single asset and saves it to the designated path in the download directory
    async fn download_asset(
        client: &Client,
        download_dir: &PathBuf,
        place_id: &str,
        version: &str,
        url: &str,
    ) -> Result<(), Box<dyn std::error::Error>> {
        // Extract asset filename from URL
        let original_filename = match url.split('/').last() {
            Some(name) => name,
            None => {
                return Err("Failed to extract asset filename from URL".into());
            }
        };
    
        // Append ".rbxl" extension
        let asset_filename = format!("{}_{}.rbxl", version, original_filename);
    
        // Define the target directory and file path in the download directory
        let target_dir = download_dir.join(place_id);
        let target_path = target_dir.join(&asset_filename);
    
        // Create the target directory if it doesn't exist
        create_dir_all(&target_dir).await?;
    
        // Check if the file already exists to avoid re-downloading
        if target_path.exists() {
            info!("Asset already exists: {:?}", target_path);
            return Ok(());
        }
    
        // Make the HTTP GET request
        let response = client.get(url).send().await?;
    
        if !response.status().is_success() {
            return Err(format!("Failed to download {}: HTTP {}", url, response.status()).into());
        }
    
        // Stream the response bytes to the file
        let mut file = File::create(&target_path).await?;
        let mut stream = response.bytes_stream();
    
        while let Some(chunk) = stream.next().await {
            let data = chunk?;
            file.write_all(&data).await?;
        }
    
        info!("Downloaded: {:?}", target_path);
        Ok(())
    }
    

    Explanation of Changes

    1. Separate Reading and Download Directories:

      • Reading Directory (read_dir):
        let read_dir = Path::new(r"C:\Users\Lukas\Downloads\lkjhgfd2");
        
      • Download Directory (download_dir):
        let download_dir = Path::new(r"C:\Users\Lukas\Downloads\lkjhgfd3");
        
      • Ensure Download Directory Exists:
        if let Err(e) = fs::create_dir_all(&download_dir) {
            error!(
                "Failed to create download directory {}: {}",
                download_dir.display(),
                e
            );
            return;
        }
        
      • These directories are passed separately to the process_place_id function to handle reading and downloading independently.
    2. Modified process_place_id Function:

      • Updated to accept both read_dir and download_dir as parameters:
        async fn process_place_id(
            client: &Client,
            read_dir: &PathBuf,
            download_dir: &PathBuf,
            place_id: &str,
        ) -> Result<(), Box<dyn std::error::Error>> {
            // Function implementation
        }
        
      • Reading index.txt from read_dir:
        let index_path = read_dir.join(place_id).join("index.txt");
        
    3. Modified download_asset Function:

      • Updated to save files in the download_dir with the {version}_{hash}.rbxl naming convention:
        let asset_filename = format!("{}_{}.rbxl", version, original_filename);
        
      • Target Directory and Path:
        let target_dir = download_dir.join(place_id);
        let target_path = target_dir.join(&asset_filename);
        
      • No Subdirectories for Versions:
        • All assets for a place_id are saved directly within download_dir/place_id/ with the filename including the version.
    4. Concurrency and Efficiency:

      • Separate Concurrency Limits:
        • PlaceID Processing: Limited to concurrency_limit = 5.
        • Asset Downloads per PlaceID: Limited to asset_concurrency = 10.
      • These limits can be adjusted based on your system's capabilities and network bandwidth.
    5. Logging Enhancements:

      • Added more descriptive log messages to track progress and errors effectively.

    Directory Structure Example

    • Reading Directory (lkjhgfd2):

      C:\Users\Lukas\Downloads\lkjhgfd2\
          ├─ 11263735303\
              └─ index.txt
          ├─ 11263873956\
              └─ index.txt
          └─ ... (other placeIDs)
      
    • Download Directory (lkjhgfd3):

      C:\Users\Lukas\Downloads\lkjhgfd3\
          ├─ 11263735303\
              ├─ 1_hash1.rbxl
              ├─ 2_hash2.rbxl
              └─ ... (other assets)
          ├─ 11263873956\
              ├─ 1_hashA.rbxl
              ├─ 2_hashB.rbxl
              └─ ... (other assets)
          └─ ... (other placeIDs)
      

    Additional Recommendations

    1. Error Handling:

      • The current implementation logs errors but continues processing. Depending on your requirements, you might want to implement retry mechanisms for transient errors.
    2. Configuration:

      • Consider externalizing the directories and concurrency limits into a configuration file or environment variables for greater flexibility.
    3. Progress Tracking:

      • Implement progress indicators or summaries to monitor the overall download progress, especially for large numbers of assets.
    4. Filename Sanitization:

      • Ensure that the filenames generated are valid for the target filesystem. If version or hash can contain characters invalid in filenames, sanitize them accordingly.
    5. Optional Features:

      • Resume Capability: Implement functionality to resume downloads if the process is interrupted.
      • Parallelism Optimization: Adjust concurrency limits based on runtime feedback for optimal performance.

    Conclusion

    With these modifications, your program now:

    • Reads index.txt files from a separate directory (lkjhgfd2).
    • Downloads assets to a different directory (lkjhgfd3).
    • Uses a flat file naming convention ({version}_{hash}.rbxl) to avoid creating unnecessary subdirectories.

    This should help in organizing your downloads more effectively and prevent confusion between reading and downloading directories.