Write a code for multiple .nc file of rainfall data of every year read every year .Nc…
- User
Write a code for multiple .nc file of rainfall data of every year read every year .Nc File and make monthly and annual average export the data in shapefile
Assistant
I'll help you create a Python script to process multiple NetCDF rainfall files, calculate monthly and annual averages, and export the results to shapefiles. Here's a comprehensive solution:
import xarray as xr import pandas as pd import numpy as np import geopandas as gpd from shapely.geometry import Point import glob import os from datetime import datetime import warnings warnings.filterwarnings('ignore') def process_rainfall_data(input_directory, output_directory): """ Process multiple NetCDF rainfall files to calculate monthly and annual averages and export results to shapefiles. Parameters: input_directory (str): Path to directory containing .nc files output_directory (str): Path to directory for output shapefiles """ # Create output directory if it doesn't exist os.makedirs(output_directory, exist_ok=True) # Get all .nc files in the input directory nc_files = glob.glob(os.path.join(input_directory, "*.nc")) if not nc_files: print("No .nc files found in the specified directory") return print(f"Found {len(nc_files)} NetCDF files") # Dictionary to store data for each year yearly_data = {} # Process each NetCDF file for file_path in nc_files: print(f"Processing: {os.path.basename(file_path)}") try: # Open the NetCDF file ds = xr.open_dataset(file_path) # Print dataset info for first file if file_path == nc_files[0]: print("Dataset variables:", list(ds.variables.keys())) print("Dataset dimensions:", ds.dims) # Identify rainfall variable (common names) rainfall_vars = ['precipitation', 'precip', 'rainfall', 'rain', 'pr', 'tp'] rainfall_var = None for var in rainfall_vars: if var in ds.variables: rainfall_var = var break if rainfall_var is None: print(f"Warning: No rainfall variable found in {file_path}") continue # Get the rainfall data rainfall_data = ds[rainfall_var] # Get coordinates if 'lat' in ds.coords: lat_coord, lon_coord = 'lat', 'lon' elif 'latitude' in ds.coords: lat_coord, lon_coord = 'latitude', 'longitude' else: print(f"Warning: Cannot find lat/lon coordinates in {file_path}") continue # Get time coordinate time_coord = 'time' if 'time' not in ds.coords: time_coords = [coord for coord in ds.coords if 'time' in coord.lower()] if time_coords: time_coord = time_coords[0] else: print(f"Warning: No time coordinate found in {file_path}") continue # Convert to pandas datetime if needed if not isinstance(ds[time_coord].values[0], (pd.Timestamp, datetime)): ds = ds.assign_coords({time_coord: pd.to_datetime(ds[time_coord].values)}) # Group by year years = ds.groupby(f'{time_coord}.year') for year, year_data in years: if year not in yearly_data: yearly_data[year] = [] yearly_data[year].append(year_data[rainfall_var]) ds.close() except Exception as e: print(f"Error processing {file_path}: {str(e)}") continue # Process each year's data for year, data_list in yearly_data.items(): print(f"Processing year {year}") # Combine data from multiple files for the same year if needed if len(data_list) > 1: year_data = xr.concat(data_list, dim=time_coord) else: year_data = data_list[0] # Calculate monthly averages monthly_avg = year_data.groupby(f'{time_coord}.month').mean() # Calculate annual average annual_avg = year_data.mean(dim=time_coord) # Create monthly shapefiles create_shapefile_from_data(monthly_avg, lat_coord, lon_coord, output_directory, f"monthly_rainfall_{year}", is_monthly=True) # Create annual shapefile create_shapefile_from_data(annual_avg, lat_coord, lon_coord, output_directory, f"annual_rainfall_{year}", is_monthly=False) def create_shapefile_from_data(data_array, lat_coord, lon_coord, output_dir, filename, is_monthly=False): """ Create shapefile from xarray data """ try: # Get coordinates lats = data_array[lat_coord].values lons = data_array[lon_coord].values # Create meshgrid for coordinates lon_grid, lat_grid = np.meshgrid(lons, lats) # Flatten coordinates lat_flat = lat_grid.flatten() lon_flat = lon_grid.flatten() # Create list to store features features = [] if is_monthly: # Process monthly data for month in range(1, 13): if month in data_array.month: month_data = data_array.sel(month=month).values.flatten() for i, (lat, lon, value) in enumerate(zip(lat_flat, lon_flat, month_data)): if not np.isnan(value): features.append({ 'geometry': Point(lon, lat), 'month': month, 'rainfall': float(value), 'point_id': i }) else: # Process annual data annual_data = data_array.values.flatten() for i, (lat, lon, value) in enumerate(zip(lat_flat, lon_flat, annual_data)): if not np.isnan(value): features.append({ 'geometry': Point(lon, lat), 'rainfall': float(value), 'point_id': i }) # Create GeoDataFrame if features: gdf = gpd.GeoDataFrame(features, crs='EPSG:4326') # Save to shapefile output_path = os.path.join(output_dir, f"{filename}.shp") gdf.to_file(output_path) print(f"Saved: {output_path}") else: print(f"No valid data found for {filename}") except Exception as e: print(f"Error creating shapefile for {filename}: {str(e)}") def create_summary_statistics(input_directory, output_directory): """ Create summary statistics across all years """ print("Creating summary statistics...") # Get all annual shapefiles annual_shapefiles = glob.glob(os.path.join(output_directory, "annual_rainfall_*.shp")) if not annual_shapefiles: print("No annual shapefiles found for summary statistics") return # Read all annual data all_annual_data = [] for shp_file in annual_shapefiles: year = int(os.path.basename(shp_file).split('_')[-1].split('.')[0]) gdf = gpd.read_file(shp_file) gdf['year'] = year all_annual_data.append(gdf) # Combine all data combined_gdf = gpd.GeoDataFrame(pd.concat(all_annual_data, ignore_index=True)) # Calculate statistics by point stats_by_point = combined_gdf.groupby('point_id').agg({ 'rainfall': ['mean', 'std', 'min', 'max', 'count'] }).round(2) # Flatten column names stats_by_point.columns = ['_'.join(col).strip() for col in stats_by_point.columns] # Get geometry for each point (using first occurrence) point_geometries = combined_gdf.groupby('point_id')['geometry'].first() # Create summary GeoDataFrame summary_gdf = gpd.GeoDataFrame(stats_by_point, geometry=point_geometries, crs='EPSG:4326') # Save summary shapefile summary_path = os.path.join(output_directory, "rainfall_summary_statistics.shp") summary_gdf.to_file(summary_path) print(f"Saved summary statistics: {summary_path}") # Main execution if __name__ == "__main__": # Set your input and output directories input_directory = "path/to/your/nc/files" # Change this to your input directory output_directory = "path/to/output/shapefiles" # Change this to your output directory # Process the rainfall data process_rainfall_data(input_directory, output_directory) # Create summary statistics create_summary_statistics(input_directory, output_directory) print("Processing complete!")Here's also a simplified version if you need something more basic:
import xarray as xr import geopandas as gpd import pandas as pd import numpy as np from shapely.geometry import Point import glob import os def simple_rainfall_processor(input_dir, output_dir): """ Simplified version for basic rainfall processing """ os.makedirs(output_dir, exist_ok=True) # Get all .nc files nc_files = glob.glob(os.path.join(input_dir, "*.nc")) for file_path in nc_files: print(f"Processing: {os.path.basename(file_path)}") # Open dataset ds = xr.open_dataset(file_path) # Assume standard variable names (adjust as needed) rainfall = ds['precipitation'] # Change variable name as needed # Calculate monthly and annual averages monthly_avg = rainfall.groupby('time.month').mean() annual_avg = rainfall.mean(dim='time') # Get year from filename or dataset year = pd.to_datetime(ds.time.values[0]).year # Convert to points and save save_as_points(annual_avg, f"annual_{year}", output_dir) save_as_points(monthly_avg, f"monthly_{year}", output_dir, is_monthly=True) ds.close() def save_as_points(data_array, filename, output_dir, is_monthly=False): """ Convert xarray to point shapefile """ # Create coordinate meshgrid lons, lats = np.meshgrid(data_array.lon, data_array.lat) points = [] for i in range(len(data_array.lat)): for j in range(len(data_array.lon)): if is_monthly: for month in range(1, 13): if month in data_array.month: value = data_array.sel(month=month).values[i, j] if not np.isnan(value): points.append({ 'geometry': Point(lons[i, j], lats[i, j]), 'month': month, 'rainfall': float(value) }) else: value = data_array.values[i, j] if not np.isnan(value): points.append({ 'geometry': Point(lons[i, j], lats[i, j]), 'rainfall': float(value) }) # Create and save GeoDataFrame if points: gdf = gpd.GeoDataFrame(points, crs='EPSG:4326') gdf.to_file(os.path.join(output_dir, f"{filename}.shp")) print(f"Saved: {filename}.shp") # Usage input_directory = "your/input/path" output_directory = "your/output/path" simple_rainfall_processor(input_directory, output_directory)Required packages:
pip install xarray netcdf4 geopandas pandas numpy shapelyKey features of this code:
- Automatic file detection: Finds all .nc files in the specified directory
- Flexible variable naming: Handles common rainfall variable names
- Monthly and annual averages: Calculates both types of statistics
- Shapefile export: Creates point shapefiles with rainfall values
- Error handling: Continues processing even if some files fail
- Summary statistics: Optional function to create overall statistics
To use this code:
- Install the required packages
- Update the
input_directoryandoutput_directorypaths - Adjust variable names in the code if your NetCDF files use different naming conventions
- Run the script
The code will create separate shapefiles for each year's monthly and annual averages, plus a summary statistics file combining all years.
- User
ask to browse the input folder where all .nc files are present
Assistant
I'll modify the code to include a file browser dialog that lets you select the input folder containing your .nc files. Here's the updated version:
import xarray as xr import pandas as pd import numpy as np import geopandas as gpd from shapely.geometry import Point import glob import os from datetime import datetime import warnings import tkinter as tk from tkinter import filedialog, messagebox warnings.filterwarnings('ignore') def browse_input_folder(): """ Open a dialog to browse and select the input folder containing .nc files """ root = tk.Tk() root.withdraw() # Hide the main window # Open folder selection dialog folder_path = filedialog.askdirectory( title="Select folder containing NetCDF (.nc) files", initialdir=os.getcwd() ) root.destroy() if folder_path: # Check if folder contains .nc files nc_files = glob.glob(os.path.join(folder_path, "*.nc")) if nc_files: print(f"Selected folder: {folder_path}") print(f"Found {len(nc_files)} NetCDF files:") for file in nc_files[:5]: # Show first 5 files print(f" - {os.path.basename(file)}") if len(nc_files) > 5: print(f" ... and {len(nc_files) - 5} more files") return folder_path else: messagebox.showwarning("No NetCDF Files", f"No .nc files found in the selected folder:\n{folder_path}") return None else: print("No folder selected") return None def browse_output_folder(): """ Open a dialog to browse and select the output folder for shapefiles """ root = tk.Tk() root.withdraw() # Hide the main window # Open folder selection dialog folder_path = filedialog.askdirectory( title="Select output folder for shapefiles", initialdir=os.getcwd() ) root.destroy() if folder_path: print(f"Output folder: {folder_path}") return folder_path else: print("No output folder selected") return None def process_rainfall_data(input_directory, output_directory): """ Process multiple NetCDF rainfall files to calculate monthly and annual averages and export results to shapefiles. """ # Create output directory if it doesn't exist os.makedirs(output_directory, exist_ok=True) # Get all .nc files in the input directory nc_files = glob.glob(os.path.join(input_directory, "*.nc")) if not nc_files: print("No .nc files found in the specified directory") return print(f"\nProcessing {len(nc_files)} NetCDF files...") # Dictionary to store data for each year yearly_data = {} # Process each NetCDF file for i, file_path in enumerate(nc_files, 1): print(f"[{i}/{len(nc_files)}] Processing: {os.path.basename(file_path)}") try: # Open the NetCDF file ds = xr.open_dataset(file_path) # Print dataset info for first file if file_path == nc_files[0]: print("\nDataset information:") print("Variables:", list(ds.variables.keys())) print("Dimensions:", ds.dims) print("Coordinate variables:", list(ds.coords.keys())) # Identify rainfall variable (common names) rainfall_vars = ['precipitation', 'precip', 'rainfall', 'rain', 'pr', 'tp', 'prcp'] rainfall_var = None for var in rainfall_vars: if var in ds.variables: rainfall_var = var break if rainfall_var is None: print(f"Warning: No rainfall variable found in {file_path}") print(f"Available variables: {list(ds.variables.keys())}") continue # Get the rainfall data rainfall_data = ds[rainfall_var] # Get coordinates if 'lat' in ds.coords: lat_coord, lon_coord = 'lat', 'lon' elif 'latitude' in ds.coords: lat_coord, lon_coord = 'latitude', 'longitude' elif 'y' in ds.coords and 'x' in ds.coords: lat_coord, lon_coord = 'y', 'x' else: print(f"Warning: Cannot find lat/lon coordinates in {file_path}") print(f"Available coordinates: {list(ds.coords.keys())}") continue # Get time coordinate time_coord = 'time' if 'time' not in ds.coords: time_coords = [coord for coord in ds.coords if 'time' in coord.lower()] if time_coords: time_coord = time_coords[0] else: print(f"Warning: No time coordinate found in {file_path}") continue # Convert to pandas datetime if needed try: if not isinstance(ds[time_coord].values[0], (pd.Timestamp, datetime)): ds = ds.assign_coords({time_coord: pd.to_datetime(ds[time_coord].values)}) except: print(f"Warning: Could not convert time coordinate to datetime for {file_path}") continue # Group by year years = ds.groupby(f'{time_coord}.year') for year, year_data in years: if year not in yearly_data: yearly_data[year] = [] yearly_data[year].append({ 'data': year_data[rainfall_var], 'lat_coord': lat_coord, 'lon_coord': lon_coord, 'time_coord': time_coord }) ds.close() except Exception as e: print(f"Error processing {file_path}: {str(e)}") continue # Process each year's data for year, data_list in yearly_data.items(): print(f"\nProcessing year {year}") # Use the first dataset's coordinates (assuming all have same grid) lat_coord = data_list[0]['lat_coord'] lon_coord = data_list[0]['lon_coord'] time_coord = data_list[0]['time_coord'] # Combine data from multiple files for the same year if needed if len(data_list) > 1: year_data = xr.concat([item['data'] for item in data_list], dim=time_coord) else: year_data = data_list[0]['data'] # Calculate monthly averages monthly_avg = year_data.groupby(f'{time_coord}.month').mean() # Calculate annual average annual_avg = year_data.mean(dim=time_coord) # Create monthly shapefiles create_shapefile_from_data(monthly_avg, lat_coord, lon_coord, output_directory, f"monthly_rainfall_{year}", is_monthly=True) # Create annual shapefile create_shapefile_from_data(annual_avg, lat_coord, lon_coord, output_directory, f"annual_rainfall_{year}", is_monthly=False) def create_shapefile_from_data(data_array, lat_coord, lon_coord, output_dir, filename, is_monthly=False): """ Create shapefile from xarray data """ try: # Get coordinates lats = data_array[lat_coord].values lons = data_array[lon_coord].values # Create meshgrid for coordinates lon_grid, lat_grid = np.meshgrid(lons, lats) # Flatten coordinates lat_flat = lat_grid.flatten() lon_flat = lon_grid.flatten() # Create list to store features features = [] if is_monthly: # Process monthly data for month in range(1, 13): if month in data_array.month: month_data = data_array.sel(month=month).values.flatten() for i, (lat, lon, value) in enumerate(zip(lat_flat, lon_flat, month_data)): if not np.isnan(value): features.append({ 'geometry': Point(lon, lat), 'month': month, 'rainfall': float(value), 'point_id': i }) else: # Process annual data annual_data = data_array.values.flatten() for i, (lat, lon, value) in enumerate(zip(lat_flat, lon_flat, annual_data)): if not np.isnan(value): features.append({ 'geometry': Point(lon, lat), 'rainfall': float(value), 'point_id': i }) # Create GeoDataFrame if features: gdf = gpd.GeoDataFrame(features, crs='EPSG:4326') # Save to shapefile output_path = os.path.join(output_dir, f"{filename}.shp") gdf.to_file(output_path) print(f" Saved: {os.path.basename(output_path)}") else: print(f" No valid data found for {filename}") except Exception as e: print(f" Error creating shapefile for {filename}: {str(e)}") def main(): """ Main function with interactive folder selection """ print("=== NetCDF Rainfall Data Processor ===") print("This tool will process .nc files and create shapefiles with monthly and annual rainfall averages.\n") # Browse for input folder print("Step 1: Select input folder containing .nc files") input_directory = browse_input_folder() if not input_directory: print("No input folder selected. Exiting...") return # Browse for output folder print("\nStep 2: Select output folder for shapefiles") output_directory = browse_output_folder() if not output_directory: print("No output folder selected. Exiting...") return # Confirm processing print(f"\nInput folder: {input_directory}") print(f"Output folder: {output_directory}") proceed = input("\nProceed with processing? (y/n): ").lower().strip() if proceed in ['y', 'yes']: print("\nStarting processing...") process_rainfall_data(input_directory, output_directory) print(f"\nProcessing complete! Check the output folder: {output_directory}") else: print("Processing cancelled.") # Alternative function for command line usage def process_with_paths(input_path=None, output_path=None): """ Process with specified paths (for command line usage) """ if input_path is None: input_path = input("Enter input folder path: ").strip() if output_path is None: output_path = input("Enter output folder path: ").strip() if os.path.exists(input_path): process_rainfall_data(input_path, output_path) else: print(f"Input path does not exist: {input_path}") if __name__ == "__main__": # Check if running in an environment that supports GUI try: # Try to create a test Tkinter window test_root = tk.Tk() test_root.withdraw() test_root.destroy() # If successful, run the GUI version main() except Exception as e: print("GUI not available. Using command line interface...") print(f"Error: {e}") # Fallback to command line input process_with_paths()Key features of the updated code:
- Interactive folder browsing: Uses
tkinter.filedialogto open folder selection dialogs - Input validation: Checks if the selected folder contains .nc files
- Progress tracking: Shows which file is being processed
- Detailed information: Displays dataset structure for the first file
- Error handling: Continues processing even if some files fail
- Fallback option: If GUI is not available, falls back to command line input
Required packages:
pip install xarray netcdf4 geopandas pandas numpy shapely tkinterHow to use:
- Run the script
- A dialog will open asking you to select the input folder containing your .nc files
- Another dialog will ask you to select the output folder for the shapefiles
- The script will show you the selected paths and ask for confirmation
- Processing will begin automatically
If you prefer command line usage without GUI:
# Simple command line version if __name__ == "__main__": input_dir = input("Enter path to folder with .nc files: ").strip() output_dir = input("Enter path for output shapefiles: ").strip() if os.path.exists(input_dir): process_rainfall_data(input_dir, output_dir) else: print("Input directory does not exist!")The script will automatically detect the structure of your NetCDF files and handle different variable naming conventions commonly used for rainfall data.
- Interactive folder browsing: Uses
Other models, same or similar prompt
Responses were produced in different, uncontrolled contexts (system prompts, settings, prior turns). This is not a benchmark.