use std::fs; use std::path::{Path, PathBuf}; use std::time::Duration; use…
- User
use std::fs; use std::path::{Path, PathBuf}; use std::time::Duration; use tokio::fs::{create_dir_all, File}; use tokio::io::AsyncWriteExt; use reqwest::Client; use futures::stream::{self, StreamExt}; use tracing::{error, info};
#[tokio::main] async fn main() { tracing_subscriber::fmt::init(); // Base directory where the downloads will be saved let base_dir = Path::new(r"C:\Users\Lukas\Downloads\lkjhgfd2");
// List of placeIDs let place_ids = vec![ "10789565990", "11181971058", "11181982436", "11182034319", "11182037286", "11263735303", "11263873956", "12647442701", "33564694", "4568576845", "4737314314", "4748963513", "4956810680", "4998492434", "5000291183", "5063342777", "5229799451", "5400427674", "5465723678", "5914914682", "5972110179", "6087359275", "6701464352", "6767882272", "6768240609", "6783491168", "7162161171", "7730658758", "9048939285", ]; // Initialize HTTP client with timeout and other configurations let client = Client::builder() .timeout(Duration::from_secs(30)) .build() .expect("Failed to build HTTP client"); // Process each placeID concurrently with a limit on concurrency let concurrency_limit = 5; // Adjust based on your network capability stream::iter(place_ids) .for_each_concurrent(concurrency_limit, |place_id| { let client = client.clone(); let base_dir = base_dir.to_path_buf(); async move { if let Err(e) = process_place_id(&client, &base_dir, place_id).await { error!("Error processing placeID {}: {}", place_id, e); } } }) .await; info!("All downloads completed.");}
/// Processes a single placeID: /// - Reads index.txt /// - Parses entries /// - Downloads assets async fn process_place_id(client: &Client, base_dir: &PathBuf, place_id: &str) -> Result<(), Box> { info!("Processing placeID: {}", place_id);
// Path to index.txt let index_path = base_dir.join(place_id).join("index.txt"); // Read index.txt content let index_content = match fs::read_to_string(&index_path) { Ok(content) => content, Err(e) => { error!("Failed to read {}: {}", index_path.display(), e); return Err(Box::new(e)); } }; // Parse index.txt lines let assets: Vec<(String, String)> = index_content .lines() .filter_map(|line| { let parts: Vec<&str> = line.split('|').map(|s| s.trim()).collect(); if parts.len() == 2 { Some((parts[0].to_string(), parts[1].to_string())) } else { None } }) .collect(); info!("---------------------"); info!("Found {} assets for placeID {}", assets.len(), place_id); info!("---------------------"); // Define concurrency for asset downloads per placeID let asset_concurrency = 10; // Adjust as needed // Process asset downloads concurrently stream::iter(assets) .for_each_concurrent(asset_concurrency, |(version, url)| { let client = client.clone(); let base_dir = base_dir.clone(); let place_id = place_id.to_string(); async move { if let Err(e) = download_asset(&client, &base_dir, &place_id, &version, &url).await { error!("Failed to download asset {} for placeID {}: {}", url, place_id, e); } } }) .await; info!("Completed placeID: {}", place_id); Ok(())}
/// Downloads a single asset and saves it to the designated path async fn download_asset( client: &Client, base_dir: &PathBuf, place_id: &str, version: &str, url: &str, ) -> Result<(), Box> { // Extract asset filename from URL let original_filename = match url.split('/').last() { Some(name) => name, None => { return Err("Failed to extract asset filename from URL".into()); } };
// Append ".rbxl" extension let asset_filename = format!("{}.rbxl", original_filename); // Define the target directory and file path let target_dir = base_dir.join(place_id).join(version); let target_path = target_dir.join(&asset_filename); // Create directories if they don't exist create_dir_all(&target_dir).await?; // Check if the file already exists to avoid re-downloading if target_path.exists() { info!("Asset already exists: {:?}", target_path); return Ok(()); } // Make the HTTP GET request let response = client.get(url).send().await?; if !response.status().is_success() { return Err(format!("Failed to download {}: HTTP {}", url, response.status()).into()); } // Stream the response bytes to the file let mut file = File::create(&target_path).await?; let mut stream = response.bytes_stream(); while let Some(chunk) = stream.next().await { let data = chunk?; file.write_all(&data).await?; } info!("Downloaded: {:?}", target_path); Ok(())}
Make it so it downloads somewhere else than the path that has the index.txts and stuff.
Make it so it doesn't create files like C:\Users\Lukas\Downloads\lkjhgfd2\11263735303\1{hash}.rbxl More like C:\Users\Lukas\Downloads\lkjhgfd2\11263735303\1_{hash}.rbxl
e.x if my path for reading the index.txts was C:\Users\Lukas\Downloads\lkjhgfd2
my path for the downloads urls would be like C:\Users\Lukas\Downloads\lkjhgfd3\11263735303\1_{hash}.rbxl to remove confusion
- Assistant
Sure! Let's modify your Rust program to meet the following requirements:
-
Separate Directories for Reading and Downloading:
- Reading Directory (e.g.,
lkjhgfd2): This is where yourindex.txtfiles are located. - Download Directory (e.g.,
lkjhgfd3): This is where the downloaded assets will be saved.
- Reading Directory (e.g.,
-
Custom File Naming Convention:
- Instead of creating separate subdirectories for each version, the files will be named using the format
{version}_{hash}.rbxland saved directly within thedownload_dir/place_iddirectory.
- Instead of creating separate subdirectories for each version, the files will be named using the format
Updated Code
Below is the updated Rust code with the necessary modifications and comments explaining the changes:
use std::fs; use std::path::{Path, PathBuf}; use std::time::Duration; use tokio::fs::{create_dir_all, File}; use tokio::io::AsyncWriteExt; use reqwest::Client; use futures::stream::{self, StreamExt}; use tracing::{error, info}; #[tokio::main] async fn main() { // Initialize the tracing subscriber for logging tracing_subscriber::fmt::init(); // Define the base directory where index.txt files are located (reading directory) let read_dir = Path::new(r"C:\Users\Lukas\Downloads\lkjhgfd2"); // Define the base directory where downloads will be saved (download directory) let download_dir = Path::new(r"C:\Users\Lukas\Downloads\lkjhgfd3"); // Ensure the download directory exists if let Err(e) = fs::create_dir_all(&download_dir) { error!( "Failed to create download directory {}: {}", download_dir.display(), e ); return; } // List of placeIDs let place_ids = vec![ "10789565990", "11181971058", "11181982436", "11182034319", "11182037286", "11263735303", "11263873956", "12647442701", "33564694", "4568576845", "4737314314", "4748963513", "4956810680", "4998492434", "5000291183", "5063342777", "5229799451", "5400427674", "5465723678", "5914914682", "5972110179", "6087359275", "6701464352", "6767882272", "6768240609", "6783491168", "7162161171", "7730658758", "9048939285", ]; // Initialize HTTP client with timeout and other configurations let client = Client::builder() .timeout(Duration::from_secs(30)) .build() .expect("Failed to build HTTP client"); // Process each placeID concurrently with a limit on concurrency let concurrency_limit = 5; // Adjust based on your network capability stream::iter(place_ids) .for_each_concurrent(concurrency_limit, |place_id| { let client = client.clone(); let read_dir = read_dir.to_path_buf(); let download_dir = download_dir.to_path_buf(); async move { if let Err(e) = process_place_id(&client, &read_dir, &download_dir, place_id).await { error!("Error processing placeID {}: {}", place_id, e); } } }) .await; info!("All downloads completed."); } /// Processes a single placeID: /// - Reads index.txt from the reading directory /// - Parses entries /// - Downloads assets to the download directory async fn process_place_id( client: &Client, read_dir: &PathBuf, download_dir: &PathBuf, place_id: &str, ) -> Result<(), Box<dyn std::error::Error>> { info!("Processing placeID: {}", place_id); // Path to index.txt in the reading directory let index_path = read_dir.join(place_id).join("index.txt"); // Read index.txt content let index_content = match fs::read_to_string(&index_path) { Ok(content) => content, Err(e) => { error!("Failed to read {}: {}", index_path.display(), e); return Err(Box::new(e)); } }; // Parse index.txt lines into a vector of (version, url) tuples let assets: Vec<(String, String)> = index_content .lines() .filter_map(|line| { let parts: Vec<&str> = line.split('|').map(|s| s.trim()).collect(); if parts.len() == 2 { Some((parts[0].to_string(), parts[1].to_string())) } else { None } }) .collect(); info!("---------------------"); info!("Found {} assets for placeID {}", assets.len(), place_id); info!("---------------------"); // Define concurrency for asset downloads per placeID let asset_concurrency = 10; // Adjust as needed // Process asset downloads concurrently stream::iter(assets) .for_each_concurrent(asset_concurrency, |(version, url)| { let client = client.clone(); let download_dir = download_dir.clone(); let place_id = place_id.to_string(); async move { if let Err(e) = download_asset(&client, &download_dir, &place_id, &version, &url).await { error!( "Failed to download asset {} for placeID {}: {}", url, place_id, e ); } } }) .await; info!("Completed placeID: {}", place_id); Ok(()) } /// Downloads a single asset and saves it to the designated path in the download directory async fn download_asset( client: &Client, download_dir: &PathBuf, place_id: &str, version: &str, url: &str, ) -> Result<(), Box<dyn std::error::Error>> { // Extract asset filename from URL let original_filename = match url.split('/').last() { Some(name) => name, None => { return Err("Failed to extract asset filename from URL".into()); } }; // Append ".rbxl" extension let asset_filename = format!("{}_{}.rbxl", version, original_filename); // Define the target directory and file path in the download directory let target_dir = download_dir.join(place_id); let target_path = target_dir.join(&asset_filename); // Create the target directory if it doesn't exist create_dir_all(&target_dir).await?; // Check if the file already exists to avoid re-downloading if target_path.exists() { info!("Asset already exists: {:?}", target_path); return Ok(()); } // Make the HTTP GET request let response = client.get(url).send().await?; if !response.status().is_success() { return Err(format!("Failed to download {}: HTTP {}", url, response.status()).into()); } // Stream the response bytes to the file let mut file = File::create(&target_path).await?; let mut stream = response.bytes_stream(); while let Some(chunk) = stream.next().await { let data = chunk?; file.write_all(&data).await?; } info!("Downloaded: {:?}", target_path); Ok(()) }Explanation of Changes
-
Separate Reading and Download Directories:
- Reading Directory (
read_dir):let read_dir = Path::new(r"C:\Users\Lukas\Downloads\lkjhgfd2"); - Download Directory (
download_dir):let download_dir = Path::new(r"C:\Users\Lukas\Downloads\lkjhgfd3"); - Ensure Download Directory Exists:
if let Err(e) = fs::create_dir_all(&download_dir) { error!( "Failed to create download directory {}: {}", download_dir.display(), e ); return; } - These directories are passed separately to the
process_place_idfunction to handle reading and downloading independently.
- Reading Directory (
-
Modified
process_place_idFunction:- Updated to accept both
read_diranddownload_diras parameters:async fn process_place_id( client: &Client, read_dir: &PathBuf, download_dir: &PathBuf, place_id: &str, ) -> Result<(), Box<dyn std::error::Error>> { // Function implementation } - Reading
index.txtfromread_dir:let index_path = read_dir.join(place_id).join("index.txt");
- Updated to accept both
-
Modified
download_assetFunction:- Updated to save files in the
download_dirwith the{version}_{hash}.rbxlnaming convention:let asset_filename = format!("{}_{}.rbxl", version, original_filename); - Target Directory and Path:
let target_dir = download_dir.join(place_id); let target_path = target_dir.join(&asset_filename); - No Subdirectories for Versions:
- All assets for a
place_idare saved directly withindownload_dir/place_id/with the filename including the version.
- All assets for a
- Updated to save files in the
-
Concurrency and Efficiency:
- Separate Concurrency Limits:
- PlaceID Processing: Limited to
concurrency_limit = 5. - Asset Downloads per PlaceID: Limited to
asset_concurrency = 10.
- PlaceID Processing: Limited to
- These limits can be adjusted based on your system's capabilities and network bandwidth.
- Separate Concurrency Limits:
-
Logging Enhancements:
- Added more descriptive log messages to track progress and errors effectively.
Directory Structure Example
-
Reading Directory (
lkjhgfd2):C:\Users\Lukas\Downloads\lkjhgfd2\ ├─ 11263735303\ └─ index.txt ├─ 11263873956\ └─ index.txt └─ ... (other placeIDs) -
Download Directory (
lkjhgfd3):C:\Users\Lukas\Downloads\lkjhgfd3\ ├─ 11263735303\ ├─ 1_hash1.rbxl ├─ 2_hash2.rbxl └─ ... (other assets) ├─ 11263873956\ ├─ 1_hashA.rbxl ├─ 2_hashB.rbxl └─ ... (other assets) └─ ... (other placeIDs)
Additional Recommendations
-
Error Handling:
- The current implementation logs errors but continues processing. Depending on your requirements, you might want to implement retry mechanisms for transient errors.
-
Configuration:
- Consider externalizing the directories and concurrency limits into a configuration file or environment variables for greater flexibility.
-
Progress Tracking:
- Implement progress indicators or summaries to monitor the overall download progress, especially for large numbers of assets.
-
Filename Sanitization:
- Ensure that the filenames generated are valid for the target filesystem. If
versionorhashcan contain characters invalid in filenames, sanitize them accordingly.
- Ensure that the filenames generated are valid for the target filesystem. If
-
Optional Features:
- Resume Capability: Implement functionality to resume downloads if the process is interrupted.
- Parallelism Optimization: Adjust concurrency limits based on runtime feedback for optimal performance.
Conclusion
With these modifications, your program now:
- Reads
index.txtfiles from a separate directory (lkjhgfd2). - Downloads assets to a different directory (
lkjhgfd3). - Uses a flat file naming convention (
{version}_{hash}.rbxl) to avoid creating unnecessary subdirectories.
This should help in organizing your downloads more effectively and prevent confusion between reading and downloading directories.
-