import geopandas as gpd import matplotlib.pyplot as plt from geopy.geocoders import…
- User
import geopandas as gpd import matplotlib.pyplot as plt from geopy.geocoders import Nominatim import requests from bs4 import BeautifulSoup import re from tqdm import tqdm from shapely.geometry import Point import json import pandas as pd import numpy as np import rasterio import warnings from scipy.spatial import cKDTree from rasterio.transform import from_origin from rasterio.features import rasterize import os import rasterio.mask import matplotlib.colors as mcolors import matplotlib.patheffects as patheffects from pykrige.uk import UniversalKriging from sklearn.linear_model import HuberRegressor from scipy.stats import zscore from sklearn.ensemble import RandomForestRegressor from sklearn.preprocessing import StandardScaler from scipy.ndimage import gaussian_filter
os.environ['PROJ_LIB'] = r'C:\Users\Lenovo\Documents\conda\Library\share\proj'
warnings.filterwarnings("ignore")
Keywords to remove
keywords_to_remove = [ r"(centrum)", "rondo", "most", "Obwodnica", "obwodnica I", "obwodnica II", "obwodnica", "- Zawiszyn", "OUD Węzeł", "PPO", "Węzeł", "N (wąwóz)", "S (most)", "stacja", "z pomiarów", "min", "max", "średnia" ]
def clean_station_name(station_name): pattern = re.compile("|".join(keywords_to_remove), re.IGNORECASE) cleaned_name = pattern.sub("", station_name).strip() return cleaned_name
def get_masovian_boundary(): url = "https://raw.githubusercontent.com/ppatrzyk/polska-geojson/master/wojewodztwa/wojewodztwa-medium.geojson" # Fetch the content using requests response = requests.get(url) if response.status_code == 200: data = response.json() # Set the CRS to EPSG:4326 when creating the GeoDataFrame gdf = gpd.GeoDataFrame.from_features(data["features"], crs="EPSG:4326") masovian = gdf[gdf['nazwa'] == 'mazowieckie'] return masovian else: print(f"Error fetching GeoJSON data: {response.status_code}") return None
Global variable for Masovian boundary
masovian_boundary = get_masovian_boundary()
def is_in_masovian(lat, lon): point = gpd.GeoSeries([Point(lon, lat)], crs="EPSG:4326") masovian_geom = masovian_boundary.to_crs(epsg=2180).reset_index(drop=True) point_projected = point.to_crs(epsg=2180) return masovian_geom.contains(point_projected.buffer(100)).values[0]
def fetch_traxelektronik_data(): url = 'https://www.traxelektronik.pl/pogoda/zbiorcza.php?RejID=10' response = requests.get(url) soup = BeautifulSoup(response.text, 'html.parser')
stations = [] temperatures = [] table_rows = soup.find_all('tr')[2:] for row in table_rows: columns = row.find_all('td') if len(columns) > 1: station = columns[0].get_text(strip=True) temperature = columns[1].get_text(strip=True) if temperature != "-": cleaned_station = clean_station_name(station) stations.append(cleaned_station) temperatures.append(temperature) print(f"Fetched {len(stations)} stations with valid temperature data from Traxelektronik.") return stations, temperaturesdef fetch_netatmo_data(): url = "https://api.netatmo.com/api/getpublicdata" params = { "lat_ne": 53, "lon_ne": 22, "lat_sw": 51, "lon_sw": 19, "required_data": "temperature", "filter": "true", "access_token": "66e1e80975134797710422cd|32d400dbc222189055d99ba180d15fc2" }
response = requests.get(url, params=params) if response.status_code == 200: print("Successfully fetched data from Netatmo") else: print(f"Error fetching data: {response.status_code}") data = json.loads(response.text) stations = [] temperatures = [] coordinates = [] for station in data.get('body', []): place = station.get('place', {}) location = place.get('location') if location: lon, lat = location if is_in_masovian(lat, lon): for module, module_data in station.get('measures', {}).items(): if 'temperature' in module_data.get('type', []): res = module_data.get('res', {}) if res: latest_timestamp = max(res.keys()) temperature = res[latest_timestamp][0] city = place.get('city', 'Unknown') stations.append(city) temperatures.append(temperature) coordinates.append((lat, lon)) break print(f"Fetched {len(stations)} stations with valid temperature data from Netatmo in Masovian Voivodeship.") return stations, temperatures, coordinatesdef fetch_imgw_data(): url = "https://rafalraczynski.com.pl/imgw/dane-imgw/getJSON.php?type=table&province=14&sort=temp&order=asc" response = requests.get(url) if response.status_code == 200: data = response.json() else: print(f"Error fetching IMGW data: {response.status_code}") return [], [], []
target_stations = [ "ANDRZEJEWO", "BORKOWO", "CZARNOWO", "KAZANÓW", "RUSZKOWICE", "RYBIENKO", "RZĄŚNIK WŁOŚCIAŃSKI", "WIELGOLAS" ] stations = [] temperatures = [] station_ids = [] for item in data: statName = item.get('statName') temp = item.get('temp') if statName and temp and statName.upper() in target_stations: stations.append(statName) temperatures.append(temp) station_ids.append(item.get('statId', 'Unknown')) print(f"Fetched {len(stations)} stations from IMGW data.") return stations, temperatures, station_idsdef get_coordinates(station_name, geolocator): try: location = geolocator.geocode(f"{station_name}, Masovian Voivodeship, Poland") if location: if is_in_masovian(location.latitude, location.longitude): print(f"Geocoded '{station_name}' to {location.latitude}, {location.longitude} (in Masovian).") return (location.latitude, location.longitude), False else: print(f"'{station_name}' is not in Masovian Voivodeship.") return None, False print(f"Failed to geocode '{station_name}'.") return None, False except Exception as e: print(f"Error geocoding {station_name}: {e}") return None, False
def clean_temperature(temp): cleaned_temp = re.sub(r'[↓↑\s]', '', temp) try: return float(cleaned_temp) except ValueError: return None
def remove_nearby_duplicates(trax_coords, trax_temperatures, netatmo_coords, netatmo_temperatures, distance_threshold=0.01): trax_points = np.array([(lon, lat) for station, (lat, lon) in trax_coords.items()]) netatmo_points = np.array([(lon, lat) for lat, lon in netatmo_coords]) trax_tree = cKDTree(trax_points)
netatmo_filtered_coords = [] netatmo_filtered_temperatures = [] for netatmo_point, netatmo_temp in zip(netatmo_points, netatmo_temperatures): distance, index = trax_tree.query(netatmo_point) if distance > distance_threshold: netatmo_filtered_coords.append(tuple(netatmo_point)) netatmo_filtered_temperatures.append(netatmo_temp) return netatmo_filtered_coords, netatmo_filtered_temperaturesdef hillshade(array, azimuth=315, angle_altitude=45): # Convert angles to radians azimuth_rad = np.radians(azimuth) altitude_rad = np.radians(angle_altitude)
# Calculate gradients in x and y directions x, y = np.gradient(array) # Calculate the slope and aspect slope = np.pi/2.0 - np.arctan(np.sqrt(x*x + y*y)) aspect = np.arctan2(-x, y) # Calculate shaded relief shaded = np.sin(altitude_rad) * np.sin(slope) + np.cos(altitude_rad) * np.cos(slope) * np.cos(azimuth_rad - aspect) # Normalize the shaded relief shaded = (shaded + 1) / 2 shaded = (shaded * 255).astype(np.uint8) return shadeddef gaussian_filter_nan(image, sigma): nan_mask = np.isnan(image) image_filled = np.where(nan_mask, 0, image) weights = np.where(nan_mask, 0, 1) image_filtered = gaussian_filter(image_filled, sigma=sigma, mode='nearest') weights_filtered = gaussian_filter(weights, sigma=sigma, mode='nearest') with np.errstate(divide='ignore', invalid='ignore'): image_final = image_filtered / weights_filtered image_final[nan_mask] = np.nan return image_final
def plot_stations_with_boundary_and_interpolation( trax_stations, trax_temperatures, trax_coords, netatmo_stations, netatmo_temperatures, netatmo_coords, imgw_stations, imgw_temperatures, imgw_coords): # Process Traxelektronik data trax_filtered_stations = [] trax_filtered_temperatures = [] trax_geometries = []
# Filter and clean Traxelektronik data for station, temperature in zip(trax_stations, trax_temperatures): cleaned_station = clean_station_name(station) cleaned_temperature = clean_temperature(temperature) if station in trax_coords and cleaned_temperature is not None: lat, lon = trax_coords[station] trax_filtered_stations.append(cleaned_station) trax_filtered_temperatures.append(cleaned_temperature) trax_geometries.append(Point(lon, lat)) # Create GeoDataFrame for Traxelektronik data trax_gdf = gpd.GeoDataFrame({ 'Station': trax_filtered_stations, 'Temperature': trax_filtered_temperatures, 'Source': 'Traxelektronik' }, geometry=trax_geometries, crs="EPSG:4326") # Handle Netatmo data netatmo_filtered_coords = [] netatmo_filtered_temperatures = [] netatmo_filtered_stations = [] for station, temperature, coord in zip(netatmo_stations, netatmo_temperatures, netatmo_coords): if isinstance(temperature, (int, float)) and coord is not None: netatmo_filtered_coords.append(coord) netatmo_filtered_temperatures.append(temperature) netatmo_filtered_stations.append(station) if len(netatmo_filtered_coords) > 0: netatmo_geometries = [Point(lon, lat) for lat, lon in netatmo_filtered_coords] netatmo_gdf = gpd.GeoDataFrame({ 'Station': netatmo_filtered_stations, 'Temperature': netatmo_filtered_temperatures, 'Source': 'Netatmo' }, geometry=netatmo_geometries, crs="EPSG:4326") else: netatmo_gdf = gpd.GeoDataFrame(columns=['Station', 'Temperature', 'Source', 'geometry'], crs="EPSG:4326") # Handle IMGW data imgw_filtered_stations = [] imgw_filtered_temperatures = [] imgw_geometries = [] # Clean and process IMGW data for station, temperature in zip(imgw_stations, imgw_temperatures): cleaned_temperature = clean_temperature(temperature) if station in imgw_coords and cleaned_temperature is not None: lat, lon = imgw_coords[station] imgw_filtered_stations.append(station) imgw_filtered_temperatures.append(cleaned_temperature) imgw_geometries.append(Point(lon, lat)) # Create GeoDataFrame for IMGW data imgw_gdf = gpd.GeoDataFrame({ 'Station': imgw_filtered_stations, 'Temperature': imgw_filtered_temperatures, 'Source': 'IMGW' }, geometry=imgw_geometries, crs="EPSG:4326") # Read and clip DEM dem_data, dem_transform = read_and_clip_dem(masovian_boundary) # Read and clip Forest Cover forest_data, forest_transform = read_and_clip_forest(masovian_boundary) # Read and clip Water Bodies water_data, water_transform = read_and_clip_water_bodies(masovian_boundary) # Ensure geometries are in EPSG:4326 trax_gdf_dem_crs = trax_gdf.to_crs("EPSG:4326") netatmo_gdf_dem_crs = netatmo_gdf.to_crs("EPSG:4326") imgw_gdf_dem_crs = imgw_gdf.to_crs("EPSG:4326") # Get elevation at station points using dem_data and dem_transform trax_elevations = get_elevation_at_points( trax_gdf_dem_crs.geometry, dem_data=dem_data, dem_transform=dem_transform ) netatmo_elevations = get_elevation_at_points( netatmo_gdf_dem_crs.geometry, dem_data=dem_data, dem_transform=dem_transform ) imgw_elevations = get_elevation_at_points( imgw_gdf_dem_crs.geometry, dem_data=dem_data, dem_transform=dem_transform ) # Add elevation to the GeoDataFrames trax_gdf['Elevation'] = trax_elevations netatmo_gdf['Elevation'] = netatmo_elevations imgw_gdf['Elevation'] = imgw_elevations # Get tree cover density at station points using forest_data and forest_transform trax_forest = get_raster_value_at_points( trax_gdf_dem_crs.geometry, raster_data=forest_data, raster_transform=forest_transform ) netatmo_forest = get_raster_value_at_points( netatmo_gdf_dem_crs.geometry, raster_data=forest_data, raster_transform=forest_transform ) imgw_forest = get_raster_value_at_points( imgw_gdf_dem_crs.geometry, raster_data=forest_data, raster_transform=forest_transform ) # Add tree cover density to the GeoDataFrames trax_gdf['Tree_Cover'] = trax_forest netatmo_gdf['Tree_Cover'] = netatmo_forest imgw_gdf['Tree_Cover'] = imgw_forest # Get water body data at station points trax_water = get_water_body_at_points( trax_gdf_dem_crs.geometry, water_data=water_data, water_transform=water_transform ) netatmo_water = get_water_body_at_points( netatmo_gdf_dem_crs.geometry, water_data=water_data, water_transform=water_transform ) imgw_water = get_water_body_at_points( imgw_gdf_dem_crs.geometry, water_data=water_data, water_transform=water_transform ) # Add water body data to the GeoDataFrames trax_gdf['Water_Body'] = trax_water netatmo_gdf['Water_Body'] = netatmo_water imgw_gdf['Water_Body'] = imgw_water # Add 'Priority' column to each GeoDataFrame trax_gdf['Priority'] = 2 netatmo_gdf['Priority'] = 1 imgw_gdf['Priority'] = 3 # Higher priority # Combine GeoDataFrames stations_gdf = gpd.GeoDataFrame(pd.concat([trax_gdf, netatmo_gdf, imgw_gdf], ignore_index=True)) stations_gdf.crs = "EPSG:4326" stations_gdf = stations_gdf.dropna(subset=['Temperature', 'Elevation', 'Tree_Cover']) # Extract coordinates stations_gdf['X'] = stations_gdf.geometry.x stations_gdf['Y'] = stations_gdf.geometry.y # Prepare data for regression X = stations_gdf[['X', 'Y', 'Elevation', 'Tree_Cover', 'Water_Body']] y = stations_gdf['Temperature'] # Normalize covariates scaler = StandardScaler() X_scaled = scaler.fit_transform(X) # Fit Random Forest Regression model print("Training Random Forest Regression model...") rf_model = RandomForestRegressor(n_estimators=100, random_state=42) rf_model.fit(X_scaled, y) # Predict temperatures and compute residuals stations_gdf['Predicted_Temperature'] = rf_model.predict(X_scaled) stations_gdf['Residual'] = stations_gdf['Temperature'] - stations_gdf['Predicted_Temperature'] # Outlier detection using Z-score stations_gdf['Residual_Z'] = zscore(stations_gdf['Residual']) z_threshold = 2 # Set your threshold stations_gdf_inliers = stations_gdf[np.abs(stations_gdf['Residual_Z']) <= z_threshold].copy() # Remove duplicates by prioritizing IMGW data data = stations_gdf_inliers.copy() # Sort by priority (higher priority first) data = data.sort_values('Priority', ascending=False) # Group by coordinates and keep the first record in each group (highest priority) data = data.groupby(['X', 'Y'], as_index=False).first() unique_points = data[['X', 'Y']].values unique_residuals = data['Residual'].values unique_elevation = data['Elevation'].values unique_tree_cover = data['Tree_Cover'].values unique_water_body = data['Water_Body'].values # Create grid in EPSG:4326 using rasterio's transform functions # Adjust grid_spacing as per your requirements grid_spacing = 0.0025 # Approximately ~5 km x_min, y_min, x_max, y_max = masovian_boundary.total_bounds # Compute the number of cols and rows with the new grid_spacing ncols = int(np.ceil((x_max - x_min) / grid_spacing)) nrows = int(np.ceil((y_max - y_min) / grid_spacing)) # Create the affine transform for the grid transform = from_origin(x_min, y_max, grid_spacing, grid_spacing) # Create arrays of row and column indices cols = np.arange(ncols) rows = np.arange(nrows) # Create a meshgrid of indices cols_indices, rows_indices = np.meshgrid(cols, rows) # Flatten the indices cols_flat = cols_indices.ravel() rows_flat = rows_indices.ravel() # Get the x and y coordinates for each grid cell center xs_flat, ys_flat = rasterio.transform.xy(transform, rows_flat, cols_flat) # Create grid dataframe grid_df = pd.DataFrame({ 'X': xs_flat, 'Y': ys_flat, 'row_idx': rows_flat, 'col_idx': cols_flat }) # Retrieve elevations from DEM grid_elevations = get_elevation_at_points( [Point(x, y) for x, y in zip(grid_df['X'], grid_df['Y'])], dem_data=dem_data, dem_transform=dem_transform ) grid_df['Elevation'] = grid_elevations # Retrieve tree cover density from forest cover raster grid_forest = get_raster_value_at_points( [Point(x, y) for x, y in zip(grid_df['X'], grid_df['Y'])], raster_data=forest_data, raster_transform=forest_transform ) grid_df['Tree_Cover'] = grid_forest # Retrieve water body data at grid points grid_water = get_water_body_at_points( [Point(x, y) for x, y in zip(grid_df['X'], grid_df['Y'])], water_data=water_data, water_transform=water_transform ) grid_df['Water_Body'] = grid_water # Remove grid points with NaN values (i.e., outside data coverage) grid_df = grid_df.dropna(subset=['Elevation', 'Tree_Cover']) # Prepare data for regression prediction X_grid = grid_df[['X', 'Y', 'Elevation', 'Tree_Cover', 'Water_Body']] X_grid_scaled = scaler.transform(X_grid) grid_df['Predicted_Temperature'] = rf_model.predict(X_grid_scaled) # Set up variogram model with adjusted parameters variogram_model = 'spherical' # Adjust these parameters based on your data; you'll need to experiment with them variogram_parameters = {'nugget': 0.2, 'sill': 1.0, 'range': 0.05} # Perform Universal Kriging print("Performing Universal Kriging interpolation for residuals with adjusted variogram parameters...") uk = UniversalKriging( x=data['X'].values, y=data['Y'].values, z=data['Residual'].values, variogram_model=variogram_model, variogram_parameters=variogram_parameters, nlags=20, verbose=True ) z, ss = uk.execute( 'points', grid_df['X'].values, grid_df['Y'].values ) grid_df['Residual'] = z grid_df['Kriging_Variance'] = ss # Final temperature estimation grid_df['Temperature'] = grid_df['Predicted_Temperature'] + grid_df['Residual'] # Apply Gaussian smoothing to the interpolated temperature grid print("Applying Gaussian smoothing to the interpolated temperature grid...") # First, create a 2D array of temperatures grid_temperature = np.full((nrows, ncols), np.nan) grid_temperature[grid_df['row_idx'], grid_df['col_idx']] = grid_df['Temperature'] # Mask the grid_temperature masovian_shape = [masovian_boundary.geometry.unary_union] # Generate mask with all_touched=True to include partial pixels mask = rasterize( masovian_shape, out_shape=(nrows, ncols), transform=transform, fill=0, all_touched=True, default_value=1, dtype='uint8' ) sigma = 1 smoothed_temperature = gaussian_filter_nan(grid_temperature, sigma=sigma) # Apply the mask to the temperature data to limit it to Masovian region smoothed_temperature = np.ma.array(smoothed_temperature, mask=mask == 0) # Also create grid_variance array (we'll skip smoothing variance for simplicity) grid_variance = np.full((nrows, ncols), np.nan) grid_variance[grid_df['row_idx'], grid_df['col_idx']] = grid_df['Kriging_Variance'] grid_variance = np.ma.array(grid_variance, mask=mask == 0) # Compute Standard Deviation from Variance grid_std_dev = np.sqrt(grid_variance) grid_std_dev = np.ma.array(grid_std_dev, mask=mask == 0) # Read the custom color scale from the CSV file color_scale_df = pd.read_csv('input/color_scale.csv') color_scale_df = color_scale_df.sort_values('value') values = color_scale_df['value'].values hex_colors = color_scale_df['color'].values # Ensure colors start with '#' hex_colors = [color if color.startswith('#') else f"#{color}" for color in hex_colors] # Create a custom colormap # Normalize the values to 0..1 vmin = values.min() vmax = values.max() normalized_values = (values - vmin) / (vmax - vmin) # Create a list of tuples (position, color) color_tuples = list(zip(normalized_values, hex_colors)) # Create a LinearSegmentedColormap cmap = mcolors.LinearSegmentedColormap.from_list('custom_cmap', color_tuples) # Set the normalization norm = mcolors.Normalize(vmin=vmin, vmax=vmax) with rasterio.open('input/masovian_dem.tif') as dem_src: # Reproject Masovian boundary to DEM CRS if necessary if masovian_boundary.crs != dem_src.crs: masovian_boundary_dem_crs = masovian_boundary.to_crs(dem_src.crs) else: masovian_boundary_dem_crs = masovian_boundary # Clip the DEM to the Masovian boundary dem_plot_data, dem_plot_transform = rasterio.mask.mask( dem_src, masovian_boundary_dem_crs.geometry, crop=True, filled=True, nodata=np.nan ) # Read the first band (assuming single-band DEM) dem_plot_data = dem_plot_data[0] # Since mask returns data as (bands, rows, cols) # Handle NaNs in the DEM data dem_plot_data = np.nan_to_num(dem_plot_data, nan=np.nanmin(dem_plot_data)) # Calculate the extent of the clipped DEM for plotting dem_left, dem_top = dem_plot_transform * (0, 0) dem_right, dem_bottom = dem_plot_transform * (dem_plot_data.shape[1], dem_plot_data.shape[0]) dem_extent = [dem_left, dem_right, dem_bottom, dem_top] # Compute the extent for plotting def get_extent(transform, array): x_min = transform.c y_max = transform.f x_max = x_min + transform.a * array.shape[1] y_min = y_max + transform.e * array.shape[0] return [x_min, x_max, y_min, y_max] hs = hillshade(dem_plot_data, azimuth=315, angle_altitude=45) extent = get_extent(transform, smoothed_temperature) dem_extent = get_extent(dem_plot_transform, dem_plot_data) # Plotting Temperature Map fig, ax = plt.subplots(figsize=(15, 15)) # Set modern and minimalistic font properties with increased size plt.rcParams.update({ 'font.size': 10, 'font.family': 'sans-serif', 'font.sans-serif': ['Arial', 'Helvetica', 'DejaVu Sans'] }) # Plot the interpolated temperature grid with custom colormap img = ax.imshow(smoothed_temperature, extent=extent, origin='upper', cmap=cmap, norm=norm, alpha=1) # Plot hillshade on top dem_img = ax.imshow(hs, extent=dem_extent, cmap='gray', origin='upper', alpha=0.3) # Overlay water bodies with blue color water_mask = np.where(water_data > 0, 1, 0) # Adjust condition based on water_data values ax.imshow( water_mask, extent=extent, origin='upper', cmap='Blues', alpha=0.3 ) # Plot Masovian Voivodeship boundary masovian_boundary.boundary.plot(ax=ax, color='black', linewidth=2) # Plot stations as smaller rectangles stations_gdf.plot( ax=ax, color='black', marker='s', # 's' for square markers (rectangles) markersize=5, # Smaller size than before label='Stations' ) # Add contours for every 0.5°C value min_temp = np.floor(smoothed_temperature.min()) max_temp = np.ceil(smoothed_temperature.max()) contour_levels = np.arange(min_temp, max_temp + 0.5, 0.5) # Plot contour lines without interrupting them contours = ax.contour( smoothed_temperature, levels=contour_levels, colors='black', linewidths=0.5, extent=extent, origin='upper' ) # Add labels without cutting the lines contour_labels = ax.clabel( contours, inline=False, fontsize=8, fmt='%0.1f', colors='white', inline_spacing=4, manual=False ) # Add black outline to contour labels for better visibility for txt in contour_labels: txt.set_path_effects([ patheffects.Stroke(linewidth=3, foreground='black'), patheffects.Normal() ]) # Add colorbar and labels cbar = fig.colorbar(img, ax=ax, label="Temperature (°C)") # Generate ticks every 5 units within the temperature range ticks = np.arange(np.floor(vmin / 5) * 5, np.ceil(vmax / 5) * 5 + 1, 5) # Adjust colorbar ticks and labels to show every 5 units cbar.set_ticks(ticks) cbar.set_ticklabels(ticks.astype(int)) ax.set_title("Temperature Interpolation in Masovian Voivodeship", fontsize=18) ax.legend() ax.set_xlabel("Longitude") ax.set_ylabel("Latitude") plt.savefig('output/output_testing.png', dpi=300, bbox_inches='tight') plt.show() print(f"Temperature interpolation plot created successfully with smoothed data.") # Plotting Error Map with Standard Deviation fig_error, ax_error = plt.subplots(figsize=(15, 15)) # Plot the kriging standard deviation grid (temperature error interpolation) cax = ax_error.imshow( grid_std_dev, extent=extent, origin='upper', cmap='RdBu', alpha=1 # Full opacity for the error data ) # Overlay the hillshade data on top with 25% transparency dem_img = ax_error.imshow( hs, extent=dem_extent, cmap='gray', origin='upper', alpha=0.5 # 25% transparency for the hillshade ) # Overlay water bodies with blue color ax_error.imshow( water_mask, extent=extent, origin='upper', cmap='Blues', alpha=0.3 ) # Overlay the Masovian Voivodeship boundary masovian_boundary.boundary.plot( ax=ax_error, color='black', linewidth=2 ) # Plot stations stations_gdf.plot( ax=ax_error, color='black', marker='s', markersize=5, label='Stations' ) # Add colorbar for the error data cbar_error = fig_error.colorbar( cax, ax=ax_error, label="Kriging Standard Deviation (°C)" ) # Set plot titles and labels ax_error.set_title( "Estimated Mean Temperature Error (Standard Deviation)", fontsize=18 ) ax_error.legend() ax_error.set_xlabel("Longitude") ax_error.set_ylabel("Latitude") # Save and show the plot plt.savefig( 'output/mean_t2m_error_testing.png', dpi=300, bbox_inches='tight' ) plt.show() print(f"Estimated mean temperature error (standard deviation) plot created successfully.") return smoothed_temperature, transformdef get_elevation_at_points(geometry, dem_data=None, dem_transform=None): from rasterio.transform import rowcol
if dem_data is not None and dem_transform is not None: elevations = [] for point in geometry: x, y = point.x, point.y try: row, col = rowcol(dem_transform, x, y) row = int(row) col = int(col) elevation = dem_data[row, col] elevations.append(elevation) except (IndexError, ValueError): elevations.append(np.nan) return elevations else: raise ValueError("Provide dem_data and dem_transform.")def get_interpolated_temperature_at_point(lat, lon, temperature_grid, transform): """ Given a latitude and longitude, return the interpolated temperature value at that point.
Parameters: - lat, lon: Latitude and longitude of the point. - temperature_grid: The interpolated temperature grid (2D numpy array). - transform: Affine transform for the grid. Returns: - Interpolated temperature value at the point, or None if point is outside the grid. """ from rasterio.transform import rowcol try: row, col = rowcol(transform, lon, lat) # Note that rasterio uses (x, y), so (lon, lat) row = int(row) col = int(col) value = temperature_grid[row, col] if np.ma.is_masked(value) or np.isnan(value): # Value is masked or NaN, meaning outside the interpolated area return None else: return value.item() except (IndexError, ValueError): return None # Point is outside the griddef get_raster_value_at_points(geometry, raster_data=None, raster_transform=None): from rasterio.transform import rowcol
if raster_data is not None and raster_transform is not None: values = [] for point in geometry: x, y = point.x, point.y try: row, col = rowcol(raster_transform, x, y) row = int(row) col = int(col) value = raster_data[row, col] values.append(value) except (IndexError, ValueError): values.append(np.nan) return values else: raise ValueError("Provide raster_data and raster_transform.")def get_water_body_at_points(geometry, water_data=None, water_transform=None): from rasterio.transform import rowcol
if water_data is not None and water_transform is not None: water_bodies = [] for point in geometry: x, y = point.x, point.y try: row, col = rowcol(water_transform, x, y) row = int(row) col = int(col) if (0 <= row < water_data.shape[0]) and (0 <= col < water_data.shape[1]): value = water_data[row, col] # Adjust the condition based on your raster's water encoding # For binary raster where 1 = water, 0 = land water_bodies.append(1 if value else 0) else: water_bodies.append(0) except (IndexError, ValueError): water_bodies.append(0) return water_bodies else: raise ValueError("Provide water_data and water_transform.")def read_and_clip_dem(masovian_boundary): dem_path = 'input/masovian_dem.tif' with rasterio.open(dem_path) as src: print("Assuming DEM is in EPSG:4326; skipping reprojection.") dem_data = src.read(1, masked=True) dem_transform = src.transform return dem_data, dem_transform
def read_and_clip_forest(masovian_boundary): forest_path = 'input/forests.tif' with rasterio.open(forest_path) as src: print("Assuming forest cover raster is in EPSG:4326; skipping reprojection.") forest_data = src.read(1, masked=True) forest_data = np.where(forest_data == src.nodata, 0, forest_data) forest_transform = src.transform return forest_data, forest_transform
def read_and_clip_water_bodies(masovian_boundary): water_path = 'input/water_bodies.tif' with rasterio.open(water_path) as src: print("Reading and clipping water bodies raster to Masovian boundaries.")
# Ensure the Masovian boundary is in the same CRS as the water raster if masovian_boundary.crs != src.crs: masovian_boundary = masovian_boundary.to_crs(src.crs) print("Reprojected Masovian boundary to match water body raster CRS.") else: print("Masovian boundary CRS matches water body raster CRS.") # Perform the clipping clipped_water, water_transform = rasterio.mask.mask( src, masovian_boundary.geometry, crop=True, filled=True, nodata=0 ) # Read the first band (assuming single-band raster) water_data = clipped_water[0] # Handle NoData values by setting them to 0 (assuming 0 represents no water) water_data = np.where(water_data == src.nodata, 0, water_data) return water_data, water_transformMain execution flow
if name == "main": # Fetch Traxelektronik data trax_stations, trax_temperatures = fetch_traxelektronik_data()
# Geocode Traxelektronik stations geolocator = Nominatim(user_agent="geoapi") trax_coords = {} progress_bar = tqdm(total=len(trax_stations), desc="Geocoding Traxelektronik Stations", unit="station") for station in trax_stations: cleaned_station = clean_station_name(station) coords, was_cleaned = get_coordinates(cleaned_station, geolocator) if coords: trax_coords[station] = coords progress_bar.update(1) progress_bar.close() # Fetch Netatmo data netatmo_stations, netatmo_temperatures, netatmo_coords = fetch_netatmo_data() # Fetch IMGW data imgw_stations, imgw_temperatures, imgw_station_ids = fetch_imgw_data() # Geocode IMGW stations imgw_coords = {} progress_bar = tqdm(total=len(imgw_stations), desc="Geocoding IMGW Stations", unit="station") for station in imgw_stations: coords, was_cleaned = get_coordinates(station, geolocator) if coords: imgw_coords[station] = coords progress_bar.update(1) progress_bar.close() # After plotting, get the interpolated temperature at specific coordinates # Coordinates to test test_lat = 51.40118 test_lon = 21.14805 # Invoke the modified function and capture the returned values smoothed_temperature, transform = plot_stations_with_boundary_and_interpolation( trax_stations, trax_temperatures, trax_coords, netatmo_stations, netatmo_temperatures, netatmo_coords, imgw_stations, imgw_temperatures, imgw_coords ) # Now, use the function to get the interpolated temperature at the test coordinates interpolated_temp = get_interpolated_temperature_at_point( test_lat, test_lon, smoothed_temperature, transform ) if interpolated_temp is not None: print(f"Interpolated temperature value at {test_lat}, {test_lon} is {interpolated_temp:.2f}°C") else: print(f"Coordinates {test_lat}, {test_lon} are outside the interpolated grid or have no data.")In that code, for some reason, because of lack of data in some places, interpolation data between weather stations becomes linear and looks "simplified", provides a sharp, linear unnatural transition between colors etc. Can you fix this? Sometimes these areas are looking fine, sometimes they're not. Based on diffirent time
- Assistant
The sharp, linear artifacts you're observing in the temperature maps are likely a result of the interpolation process in regions where data points are sparse or absent. When the interpolation algorithm lacks sufficient data in certain areas, it can produce unnatural gradients or sharp transitions, leading to the "simplified" or "linear" patterns you're seeing.
To address this issue, we need to adjust the interpolation method and parameters to better handle areas with sparse data. Here are several strategies you can implement:
-
Adjust the Variogram Model and Parameters in Kriging:
The variogram model and its parameters are crucial in kriging interpolation as they define how data points influence each other over distance. In your current code, you're using a fixed variogram model with specified parameters:
variogram_model = 'spherical' variogram_parameters = {'nugget': 0.2, 'sill': 1.0, 'range': 0.05}Issues:
- A small
rangevalue can limit the influence of data points to very short distances, causing sharp transitions in areas with sparse data. - Fixed parameters may not capture the actual spatial correlation present in your data.
Solutions:
- Increase the
rangeparameter to allow data points to influence a larger area. - Use automatic variogram fitting to let
pykrigedetermine the best variogram model and parameters based on your data.
Modified Code Example:
# Perform Universal Kriging with automatic variogram fitting print("Performing Universal Kriging interpolation for residuals with automatic variogram fitting...") uk = UniversalKriging( x=data['X'].values, y=data['Y'].values, z=data['Residual'].values, variogram_model='automatic', nlags=20, verbose=True, enable_plotting=False, # Set to True if you want to see variogram plots ) z, ss = uk.execute( 'points', grid_df['X'].values, grid_df['Y'].values ) - A small
-
Limit the Maximum Interpolation Distance:
To prevent the interpolation algorithm from making unreliable estimates far from data points, you can set a search radius in kriging:
uk = UniversalKriging( # ... other parameters ... search_radius=0.1 # Adjust this value based on your data scale )This ensures that the algorithm doesn't interpolate values in locations farther than the specified radius from known data points, reducing the likelihood of artifacts in data-sparse regions.
-
Switch to Ordinary Kriging:
If the trend surface in Universal Kriging is contributing to the artifacts, switching to Ordinary Kriging might help:
from pykrige.ok import OrdinaryKriging ok = OrdinaryKriging( x=data['X'].values, y=data['Y'].values, z=data['Residual'].values, variogram_model='automatic', nlags=20, verbose=True, enable_plotting=False, search_radius=0.1 ) z, ss = ok.execute( 'points', grid_df['X'].values, grid_df['Y'].values ) -
Use Alternative Interpolation Methods:
Kriging might not always be the best choice for your data, especially in regions with sparse coverage. Inverse Distance Weighting (IDW) is an alternative that can sometimes handle sparse data better:
from scipy.interpolate import griddata points = np.array([data['X'].values, data['Y'].values]).T values = data['Residual'].values grid_points = np.array([grid_df['X'], grid_df['Y']]).T grid_df['Residual'] = griddata( points, values, grid_points, method='linear' # Try 'cubic' or 'nearest' as well ) -
Adjust Gaussian Smoothing Parameters:
Increasing the sigma value in the Gaussian filter can help smooth out sharp transitions:
sigma = 2 # Increase from 1 to 2 smoothed_temperature = gaussian_filter_nan(grid_temperature, sigma=sigma)Be cautious, as too much smoothing can overly blur areas where data is dense.
-
Implement Data Density-Based Masking:
Create a data density map to identify areas with insufficient data. You can then:
- Mask out these areas to avoid displaying unreliable interpolated values.
- Adjust the interpolation or smoothing parameters dynamically based on data density.
Example:
from scipy.spatial import cKDTree # Create a KDTree of data points tree = cKDTree(np.c_[data['X'], data['Y']]) # For each grid point, find the distance to the nearest data point distances, _ = tree.query(np.c_[grid_df['X'], grid_df['Y']], k=1) max_distance = 0.1 # Define a threshold distance # Mask grid points farther than max_distance from any data point grid_df['Mask'] = distances > max_distance smoothed_temperature = np.ma.array(smoothed_temperature, mask=grid_df['Mask'].values.reshape(nrows, ncols)) -
Visualize Data Coverage:
Plot your data points along with the grid to assess where data is sparse:
fig, ax = plt.subplots() data.plot(ax=ax, color='red', markersize=5) masovian_boundary.boundary.plot(ax=ax, edgecolor='black') plt.show() -
Evaluate and Adjust Outlier Detection:
Ensure that the outlier detection isn't removing too much data, especially in sparse regions. You might:
-
Relax the Z-score threshold:
z_threshold = 3 # Increase from 2 to 3 -
Use robust statistical methods less sensitive to extremes.
-
-
Modify Grid Spacing:
Adjusting the grid resolution can influence interpolation:
grid_spacing = 0.005 # From 0.0025 to 0.005A coarser grid can sometimes reduce artifacts by averaging over larger areas, but it might also miss finer details.
-
Ensure Data Quality and Quantity:
If possible, include additional data sources or more data points in sparse areas to improve interpolation reliability.
Putting It All Together:
Here's how you might modify your code to implement several of these suggestions:
# ... [Previous code for data fetching and preparation] ... # Modify the variogram model and use automatic fitting print("Performing Universal Kriging interpolation for residuals with automatic variogram fitting...") uk = UniversalKriging( x=data['X'].values, y=data['Y'].values, z=data['Residual'].values, variogram_model='automatic', nlags=20, verbose=True, enable_plotting=False, # Set to True to visualize variogram search_radius=0.1 # Adjust based on data scale ) z, ss = uk.execute( 'points', grid_df['X'].values, grid_df['Y'].values ) grid_df['Residual'] = z grid_df['Kriging_Variance'] = ss # Final temperature estimation grid_df['Temperature'] = grid_df['Predicted_Temperature'] + grid_df['Residual'] # Apply increased Gaussian smoothing sigma = 2 # Increased from 1 smoothed_temperature = gaussian_filter_nan(grid_temperature, sigma=sigma) # Implement data density-based masking from scipy.spatial import cKDTree # KDTree of data points tree = cKDTree(np.c_[data['X'], data['Y']]) # Find distance to nearest data point for each grid point distances, _ = tree.query(np.c_[grid_df['X'], grid_df['Y']], k=1) max_distance = 0.1 # Define a threshold distance # Create mask for grid points beyond max_distance grid_df['Mask'] = distances > max_distance # Reshape mask to grid shape mask_array = grid_df['Mask'].values.reshape(nrows, ncols) # Apply mask to smoothed_temperature smoothed_temperature = np.ma.array( smoothed_temperature, mask=np.logical_or(mask == 0, mask_array) )Visualization Adjustments:
When plotting, you can indicate areas with insufficient data:
# Plot masked areas (e.g., with hatching or subdued colors) from matplotlib.colors import ListedColormap # Create a transparent colormap for masked areas transparent_cmap = ListedColormap(['none']) # Plot masked areas on top ax.imshow( mask_array, extent=extent, origin='upper', cmap=transparent_cmap, interpolation='none', alpha=0.3, # Adjust transparency zorder=5 # Ensure it's on top )Testing the Modifications:
After making these changes, test your script with different time frames to see if the artifacts are reduced. It's important to iteratively adjust parameters based on visual assessments and statistical checks.
Additional Considerations:
- Interpolation Uncertainty: In areas with little or no data, interpolation uncertainty is high. Consider visualizing the kriging variance or standard deviation to highlight areas where estimates are less reliable.
- Data Confidence: Communicate the limitations of the interpolated map to users, especially regarding the confidence in areas with sparse data.
- Documentation and Comments: Add comments to your code to explain why certain parameters are set, which can be helpful for future adjustments.
Conclusion:
By adjusting the variogram model, limiting interpolation distance, modifying smoothing parameters, and masking areas with insufficient data, you can reduce the unnatural artifacts in your interpolated temperature maps. It's essential to tailor these parameters to your specific dataset and to validate your results visually and statistically.
Feel free to experiment with the suggestions above, and adjust them based on the behavior of your specific data and the requirements of your analysis.
-