import geopandas as gpd import matplotlib.pyplot as plt from geopy.geocoders import…
- User
import geopandas as gpd import matplotlib.pyplot as plt from geopy.geocoders import Nominatim import requests from bs4 import BeautifulSoup import re from tqdm import tqdm from shapely.geometry import Point import json import pandas as pd import numpy as np import rasterio import warnings from scipy.spatial import cKDTree from rasterio.transform import from_origin from rasterio.features import rasterize import os import rasterio.mask import matplotlib.colors as mcolors import matplotlib.patheffects as patheffects from pykrige.uk import UniversalKriging from sklearn.linear_model import HuberRegressor from scipy.stats import zscore from sklearn.ensemble import RandomForestRegressor from sklearn.preprocessing import StandardScaler from scipy.ndimage import gaussian_filter from datetime import datetime import pytz import math
os.environ['PROJ_LIB'] = r'C:\Users\Lenovo\Documents\conda\Library\share\proj'
warnings.filterwarnings("ignore")
Keywords to remove
keywords_to_remove = [ r"(centrum)", "rondo", "most", "Obwodnica", "obwodnica I", "obwodnica II", "obwodnica", "- Zawiszyn", "OUD Węzeł", "PPO", "Węzeł", "N (wąwóz)", "S (most)", "stacja", "z pomiarów", "min", "max", "średnia" ]
def clean_station_name(station_name): pattern = re.compile("|".join(keywords_to_remove), re.IGNORECASE) cleaned_name = pattern.sub("", station_name).strip() return cleaned_name
def get_masovian_boundary(): url = "https://raw.githubusercontent.com/ppatrzyk/polska-geojson/master/wojewodztwa/wojewodztwa-medium.geojson" # Fetch the content using requests response = requests.get(url) if response.status_code == 200: data = response.json() # Set the CRS to EPSG:4326 when creating the GeoDataFrame gdf = gpd.GeoDataFrame.from_features(data["features"], crs="EPSG:4326") masovian = gdf[gdf['nazwa'] == 'mazowieckie'] return masovian else: print(f"Error fetching GeoJSON data: {response.status_code}") return None
Global variable for Masovian boundary
masovian_boundary = get_masovian_boundary()
def is_in_masovian(lat, lon): point = gpd.GeoSeries([Point(lon, lat)], crs="EPSG:4326") masovian_geom = masovian_boundary.to_crs(epsg=2180).reset_index(drop=True) point_projected = point.to_crs(epsg=2180) return masovian_geom.contains(point_projected.buffer(100)).values[0]
def fetch_traxelektronik_data(): url = 'https://www.traxelektronik.pl/pogoda/zbiorcza.php?RejID=10' response = requests.get(url) soup = BeautifulSoup(response.text, 'html.parser')
stations = [] temperatures = [] table_rows = soup.find_all('tr')[2:] for row in table_rows: columns = row.find_all('td') if len(columns) > 1: station = columns[0].get_text(strip=True) temperature = columns[1].get_text(strip=True) if temperature != "-": cleaned_station = clean_station_name(station) stations.append(cleaned_station) temperatures.append(temperature) print(f"Fetched {len(stations)} stations with valid temperature data from Traxelektronik.") return stations, temperaturesdef fetch_netatmo_data(): url = "https://api.netatmo.com/api/getpublicdata" params = { "lat_ne": 53, "lon_ne": 22, "lat_sw": 51, "lon_sw": 19, "required_data": "temperature", "filter": "true", "access_token": "66e1e80975134797710422cd|a739574f3ceeac299e7bb47cbe66b524" }
response = requests.get(url, params=params) if response.status_code == 200: print("Successfully fetched data from Netatmo") else: print(f"Error fetching data: {response.status_code}") data = json.loads(response.text) stations = [] temperatures = [] coordinates = [] for station in data.get('body', []): place = station.get('place', {}) location = place.get('location') if location: lon, lat = location if is_in_masovian(lat, lon): for module, module_data in station.get('measures', {}).items(): if 'temperature' in module_data.get('type', []): res = module_data.get('res', {}) if res: latest_timestamp = max(res.keys()) temperature = res[latest_timestamp][0] city = place.get('city', 'Unknown') stations.append(city) temperatures.append(temperature) coordinates.append((lat, lon)) break print(f"Fetched {len(stations)} stations with valid temperature data from Netatmo in Masovian Voivodeship.") return stations, temperatures, coordinatesdef fetch_imgw_data(): url = "https://rafalraczynski.com.pl/imgw/dane-imgw/getJSON.php?type=table&province=14&sort=temp&order=asc" response = requests.get(url) if response.status_code == 200: data = response.json() else: print(f"Error fetching IMGW data: {response.status_code}") return [], [], []
target_stations = [ "ANDRZEJEWO", "BORKOWO", "CZARNOWO", "KAZANÓW", "RUSZKOWICE", "RYBIENKO", "RZĄŚNIK WŁOŚCIAŃSKI", "WIELGOLAS" ] stations = [] temperatures = [] station_ids = [] for item in data: statName = item.get('statName') temp = item.get('temp') if statName and temp and statName.upper() in target_stations: stations.append(statName) temperatures.append(temp) station_ids.append(item.get('statId', 'Unknown')) print(f"Fetched {len(stations)} stations from IMGW data.") return stations, temperatures, station_idsdef get_coordinates(station_name, geolocator): try: location = geolocator.geocode(f"{station_name}, Masovian Voivodeship, Poland") if location: if is_in_masovian(location.latitude, location.longitude): print(f"Geocoded '{station_name}' to {location.latitude}, {location.longitude} (in Masovian).") return (location.latitude, location.longitude), False else: print(f"'{station_name}' is not in Masovian Voivodeship.") return None, False print(f"Failed to geocode '{station_name}'.") return None, False except Exception as e: print(f"Error geocoding {station_name}: {e}") return None, False
def clean_temperature(temp): cleaned_temp = re.sub(r'[↓↑\s]', '', temp) try: return float(cleaned_temp) except ValueError: return None
def remove_nearby_duplicates(trax_coords, trax_temperatures, netatmo_coords, netatmo_temperatures, distance_threshold=0.01): trax_points = np.array([(lon, lat) for station, (lat, lon) in trax_coords.items()]) netatmo_points = np.array([(lon, lat) for lat, lon in netatmo_coords]) trax_tree = cKDTree(trax_points)
netatmo_filtered_coords = [] netatmo_filtered_temperatures = [] for netatmo_point, netatmo_temp in zip(netatmo_points, netatmo_temperatures): distance, index = trax_tree.query(netatmo_point) if distance > distance_threshold: netatmo_filtered_coords.append(tuple(netatmo_point)) netatmo_filtered_temperatures.append(netatmo_temp) return netatmo_filtered_coords, netatmo_filtered_temperaturesdef hillshade(array, azimuth=315, angle_altitude=45): # Convert angles to radians azimuth_rad = np.radians(azimuth) altitude_rad = np.radians(angle_altitude)
# Calculate gradients in x and y directions x, y = np.gradient(array) # Calculate the slope and aspect slope = np.pi/2.0 - np.arctan(np.sqrt(x*x + y*y)) aspect = np.arctan2(-x, y) # Calculate shaded relief shaded = np.sin(altitude_rad) * np.sin(slope) + np.cos(altitude_rad) * np.cos(slope) * np.cos(azimuth_rad - aspect) # Normalize the shaded relief shaded = (shaded + 1) / 2 shaded = (shaded * 255).astype(np.uint8) return shadeddef gaussian_filter_nan(image, sigma): nan_mask = np.isnan(image) image_filled = np.where(nan_mask, 0, image) weights = np.where(nan_mask, 0, 1) image_filtered = gaussian_filter(image_filled, sigma=sigma, mode='nearest') weights_filtered = gaussian_filter(weights, sigma=sigma, mode='nearest') with np.errstate(divide='ignore', invalid='ignore'): image_final = image_filtered / weights_filtered image_final[nan_mask] = np.nan return image_final
def plot_stations_with_boundary_and_interpolation( trax_stations, trax_temperatures, trax_coords, netatmo_stations, netatmo_temperatures, netatmo_coords, imgw_stations, imgw_temperatures, imgw_coords): # Process Traxelektronik data trax_filtered_stations = [] trax_filtered_temperatures = [] trax_geometries = []
# Filter and clean Traxelektronik data for station, temperature in zip(trax_stations, trax_temperatures): cleaned_station = clean_station_name(station) cleaned_temperature = clean_temperature(temperature) if station in trax_coords and cleaned_temperature is not None: lat, lon = trax_coords[station] trax_filtered_stations.append(cleaned_station) trax_filtered_temperatures.append(cleaned_temperature) trax_geometries.append(Point(lon, lat)) # Create GeoDataFrame for Traxelektronik data trax_gdf = gpd.GeoDataFrame({ 'Station': trax_filtered_stations, 'Temperature': trax_filtered_temperatures, 'Source': 'Traxelektronik' }, geometry=trax_geometries, crs="EPSG:4326") # Handle Netatmo data netatmo_filtered_coords = [] netatmo_filtered_temperatures = [] netatmo_filtered_stations = [] for station, temperature, coord in zip(netatmo_stations, netatmo_temperatures, netatmo_coords): if isinstance(temperature, (int, float)) and coord is not None: netatmo_filtered_coords.append(coord) netatmo_filtered_temperatures.append(temperature) netatmo_filtered_stations.append(station) if len(netatmo_filtered_coords) > 0: netatmo_geometries = [Point(lon, lat) for lat, lon in netatmo_filtered_coords] netatmo_gdf = gpd.GeoDataFrame({ 'Station': netatmo_filtered_stations, 'Temperature': netatmo_filtered_temperatures, 'Source': 'Netatmo' }, geometry=netatmo_geometries, crs="EPSG:4326") else: netatmo_gdf = gpd.GeoDataFrame(columns=['Station', 'Temperature', 'Source', 'geometry'], crs="EPSG:4326") # Handle IMGW data imgw_filtered_stations = [] imgw_filtered_temperatures = [] imgw_geometries = [] # Clean and process IMGW data for station, temperature in zip(imgw_stations, imgw_temperatures): cleaned_temperature = clean_temperature(temperature) if station in imgw_coords and cleaned_temperature is not None: lat, lon = imgw_coords[station] imgw_filtered_stations.append(station) imgw_filtered_temperatures.append(cleaned_temperature) imgw_geometries.append(Point(lon, lat)) # Create GeoDataFrame for IMGW data imgw_gdf = gpd.GeoDataFrame({ 'Station': imgw_filtered_stations, 'Temperature': imgw_filtered_temperatures, 'Source': 'IMGW' }, geometry=imgw_geometries, crs="EPSG:4326") # Read and clip DEM dem_data, dem_transform = read_and_clip_dem(masovian_boundary) # Read and clip Forest Cover forest_data, forest_transform = read_and_clip_forest(masovian_boundary) # Read and clip Water Bodies water_data, water_transform = read_and_clip_water_bodies(masovian_boundary) # Read ECOSTRESS data ecostress_data, ecostress_transform = read_and_clip_ecostress(masovian_boundary) # Ensure geometries are in EPSG:4326 trax_gdf_dem_crs = trax_gdf.to_crs("EPSG:4326") netatmo_gdf_dem_crs = netatmo_gdf.to_crs("EPSG:4326") imgw_gdf_dem_crs = imgw_gdf.to_crs("EPSG:4326") # Get elevation at station points using dem_data and dem_transform trax_elevations = get_elevation_at_points( trax_gdf_dem_crs.geometry, dem_data=dem_data, dem_transform=dem_transform ) netatmo_elevations = get_elevation_at_points( netatmo_gdf_dem_crs.geometry, dem_data=dem_data, dem_transform=dem_transform ) imgw_elevations = get_elevation_at_points( imgw_gdf_dem_crs.geometry, dem_data=dem_data, dem_transform=dem_transform ) # Add elevation to the GeoDataFrames trax_gdf['Elevation'] = trax_elevations netatmo_gdf['Elevation'] = netatmo_elevations imgw_gdf['Elevation'] = imgw_elevations # Get tree cover density at station points using forest_data and forest_transform trax_forest = get_raster_value_at_points( trax_gdf_dem_crs.geometry, raster_data=forest_data, raster_transform=forest_transform ) netatmo_forest = get_raster_value_at_points( netatmo_gdf_dem_crs.geometry, raster_data=forest_data, raster_transform=forest_transform ) imgw_forest = get_raster_value_at_points( imgw_gdf_dem_crs.geometry, raster_data=forest_data, raster_transform=forest_transform ) # Add tree cover density to the GeoDataFrames trax_gdf['Tree_Cover'] = trax_forest netatmo_gdf['Tree_Cover'] = netatmo_forest imgw_gdf['Tree_Cover'] = imgw_forest # Get water body data at station points trax_water = get_water_body_at_points( trax_gdf_dem_crs.geometry, water_data=water_data, water_transform=water_transform ) netatmo_water = get_water_body_at_points( netatmo_gdf_dem_crs.geometry, water_data=water_data, water_transform=water_transform ) imgw_water = get_water_body_at_points( imgw_gdf_dem_crs.geometry, water_data=water_data, water_transform=water_transform ) # Add water body data to the GeoDataFrames trax_gdf['Water_Body'] = trax_water netatmo_gdf['Water_Body'] = netatmo_water imgw_gdf['Water_Body'] = imgw_water # Add 'Priority' column to each GeoDataFrame trax_gdf['Priority'] = 1 netatmo_gdf['Priority'] = 2 imgw_gdf['Priority'] = 3 # Higher priority # Combine GeoDataFrames stations_gdf = gpd.GeoDataFrame(pd.concat([trax_gdf, netatmo_gdf, imgw_gdf], ignore_index=True)) stations_gdf.crs = "EPSG:4326" stations_gdf = stations_gdf.dropna(subset=['Temperature', 'Elevation', 'Tree_Cover']) # Extract coordinates stations_gdf['X'] = stations_gdf.geometry.x stations_gdf['Y'] = stations_gdf.geometry.y # Prepare data for regression X = stations_gdf[['X', 'Y', 'Elevation', 'Tree_Cover', 'Water_Body']] y = stations_gdf['Temperature'] # Normalize covariates scaler = StandardScaler() X_scaled = scaler.fit_transform(X) # Fit Random Forest Regression model print("Training Random Forest Regression model...") rf_model = RandomForestRegressor(n_estimators=100, random_state=42) rf_model.fit(X_scaled, y) # Predict temperatures and compute residuals stations_gdf['Predicted_Temperature'] = rf_model.predict(X_scaled) stations_gdf['Residual'] = stations_gdf['Temperature'] - stations_gdf['Predicted_Temperature'] # Outlier detection using Z-score stations_gdf['Residual_Z'] = zscore(stations_gdf['Residual']) z_threshold = 2 # Set your threshold stations_gdf_inliers = stations_gdf[np.abs(stations_gdf['Residual_Z']) <= z_threshold].copy() # Remove duplicates by prioritizing IMGW data data = stations_gdf_inliers.copy() # Sort by priority (higher priority first) data = data.sort_values('Priority', ascending=False) # Group by coordinates and keep the first record in each group (highest priority) data = data.groupby(['X', 'Y'], as_index=False).first() unique_points = data[['X', 'Y']].values unique_residuals = data['Residual'].values unique_elevation = data['Elevation'].values unique_tree_cover = data['Tree_Cover'].values unique_water_body = data['Water_Body'].values # Create grid in EPSG:4326 using rasterio's transform functions # Adjust grid_spacing as per your requirements grid_spacing = 0.0025 # Approximately ~5 km x_min, y_min, x_max, y_max = masovian_boundary.total_bounds # Compute the number of cols and rows with the new grid_spacing ncols = int(np.ceil((x_max - x_min) / grid_spacing)) nrows = int(np.ceil((y_max - y_min) / grid_spacing)) # Create the affine transform for the grid transform = from_origin(x_min, y_max, grid_spacing, grid_spacing) # Create arrays of row and column indices cols = np.arange(ncols) rows = np.arange(nrows) # Create a meshgrid of indices cols_indices, rows_indices = np.meshgrid(cols, rows) # Flatten the indices cols_flat = cols_indices.ravel() rows_flat = rows_indices.ravel() # Get the x and y coordinates for each grid cell center xs_flat, ys_flat = rasterio.transform.xy(transform, rows_flat, cols_flat) # Create grid dataframe grid_df = pd.DataFrame({ 'X': xs_flat, 'Y': ys_flat, 'row_idx': rows_flat, 'col_idx': cols_flat }) # Retrieve elevations from DEM grid_elevations = get_elevation_at_points( [Point(x, y) for x, y in zip(grid_df['X'], grid_df['Y'])], dem_data=dem_data, dem_transform=dem_transform ) grid_df['Elevation'] = grid_elevations # Retrieve tree cover density from forest cover raster grid_forest = get_raster_value_at_points( [Point(x, y) for x, y in zip(grid_df['X'], grid_df['Y'])], raster_data=forest_data, raster_transform=forest_transform ) grid_df['Tree_Cover'] = grid_forest # Retrieve water body data at grid points grid_water = get_water_body_at_points( [Point(x, y) for x, y in zip(grid_df['X'], grid_df['Y'])], water_data=water_data, water_transform=water_transform ) grid_df['Water_Body'] = grid_water # Remove grid points with NaN values (i.e., outside data coverage) grid_df = grid_df.dropna(subset=['Elevation', 'Tree_Cover']) # Prepare data for regression prediction X_grid = grid_df[['X', 'Y', 'Elevation', 'Tree_Cover', 'Water_Body']] X_grid_scaled = scaler.transform(X_grid) grid_df['Predicted_Temperature'] = rf_model.predict(X_grid_scaled) # Set up variogram model with adjusted parameters variogram_model = 'spherical' # Adjust these parameters based on your data; you'll need to experiment with them variogram_parameters = {'nugget': 0.2, 'sill': 1.0, 'range': 0.05} # Perform Universal Kriging print("Performing Universal Kriging interpolation for residuals with adjusted variogram parameters...") uk = UniversalKriging( x=data['X'].values, y=data['Y'].values, z=data['Residual'].values, variogram_model=variogram_model, variogram_parameters=variogram_parameters, nlags=20, verbose=True ) z, ss = uk.execute( 'points', grid_df['X'].values, grid_df['Y'].values ) grid_df['Residual'] = z grid_df['Kriging_Variance'] = ss # Handle NaNs in residuals by filling them with zero grid_df['Residual'].fillna(0, inplace=True) # Final temperature estimation grid_df['Temperature'] = grid_df['Predicted_Temperature'] + grid_df['Residual'] num_nans_in_predicted_temp = grid_df['Predicted_Temperature'].isnull().sum() print(f"Number of NaNs in Predicted Temperature: {num_nans_in_predicted_temp}") # Save a copy of the temperature before ECOSTRESS adjustments grid_df['Temperature_No_ECOSTRESS'] = grid_df['Temperature'].copy() # Handle NaNs in residuals by filling them with zero grid_df['Residual'].fillna(0, inplace=True) # Final temperature estimation (before topographic cooling) grid_df['Temperature'] = grid_df['Predicted_Temperature'] + grid_df['Residual'] # Save a copy of the temperature before any adjustments grid_df['Temperature_Before_Adjustments'] = grid_df['Temperature'].copy() # Determine the current time in CET cet = pytz.timezone('CET') now_cet = datetime.now(cet) hour = now_cet.hour + now_cet.minute / 60.0 # Compute hour angle for cosine function hour_angle = ((hour % 24) / 24) * 2 * math.pi # Convert hour to radians # Compute night_weight: maximum at midnight, zero at noon night_weight = (math.cos(hour_angle) + 1) / 2 # Ranges from 1 at midnight to 0 at noon # Topographic Cooling Effect max_cooling_effect = 3.0 # Maximum cooling effect in degrees Celsius # Normalize elevation to range from 0 to 1 elevation_min = grid_df['Elevation'].min() elevation_max = grid_df['Elevation'].max() grid_df['Normalized_Elevation'] = (grid_df['Elevation'] - elevation_min) / (elevation_max - elevation_min) # Compute the topographic cooling effect grid_df['Topographic_Cooling'] = grid_df['Normalized_Elevation'] * max_cooling_effect * night_weight # Ensure topographic cooling is not applied over water bodies grid_df.loc[grid_df['Water_Body'] == 1, 'Topographic_Cooling'] = 0 # Apply the topographic cooling effect to the temperature grid_df['Temperature'] -= grid_df['Topographic_Cooling'] # ECOSTRESS Adjustments Begin Here # Retrieve ECOSTRESS LST values at grid points (same as before) grid_ecostress = get_raster_value_at_points( [Point(x, y) for x, y in zip(grid_df['X'], grid_df['Y'])], raster_data=ecostress_data, raster_transform=ecostress_transform ) grid_df['ECOSTRESS_LST'] = grid_ecostress # Handle NaNs in ECOSTRESS_LST by filling with the mean value grid_df['ECOSTRESS_LST'].fillna(grid_df['ECOSTRESS_LST'].mean(), inplace=True) # Compute ECOSTRESS anomalies (deviation from mean) ecostress_mean = grid_df['ECOSTRESS_LST'].mean() grid_df['ECOSTRESS_Anomaly'] = grid_df['ECOSTRESS_LST'] - ecostress_mean # Normalize anomalies to range [-1, 1] ecostress_anomaly_min = grid_df['ECOSTRESS_Anomaly'].min() ecostress_anomaly_max = grid_df['ECOSTRESS_Anomaly'].max() grid_df['ECOSTRESS_Anomaly_Normalized'] = ( (grid_df['ECOSTRESS_Anomaly'] - ecostress_anomaly_min) / (ecostress_anomaly_max - ecostress_anomaly_min) * 2 - 1 ) # Increase the impact of ECOSTRESS LST data amplification_factor = 3.0 # Adjust as needed # Determine the current time in CET cet = pytz.timezone('CET') now_cet = datetime.now(cet) hour = now_cet.hour + now_cet.minute / 60.0 # Compute hour angle for cosine function hour_angle = ((hour % 24) / 24) * 2 * math.pi # Convert hour to radians # Compute night_weight: maximum at midnight, zero at noon night_weight = (math.cos(hour_angle) + 1) / 2 # Ranges from 1 at midnight to 0 at noon # Define threshold for coldest areas (e.g., where ECOSTRESS anomaly is significantly low) coldest_threshold = -0.5 # You can adjust this value coldest_areas_mask = grid_df['ECOSTRESS_Anomaly_Normalized'] <= coldest_threshold # Apply anomalies with the new night_weight and only to coldest areas adjustment = grid_df['ECOSTRESS_Anomaly_Normalized'] * amplification_factor * night_weight # Apply adjustments only in the coldest areas adjustment[coldest_areas_mask] = 0 # Prevent ECOSTRESS adjustments on water bodies adjustment[grid_df['Water_Body'] == 1] = 0 # Limit maximum adjustment to avoid over-adjusting max_adjustment = 5.0 # degrees Celsius adjustment = adjustment.clip(-max_adjustment, max_adjustment) # Apply adjustments to the interpolated temperatures grid_df['Temperature'] += adjustment # Apply cooling effect on water bodies cooling_effect = -2.0 # degrees Celsius grid_df.loc[grid_df['Water_Body'] == 1, 'Temperature'] += cooling_effect # Remove Gaussian smoothing from the interpolated temperature grid # First, create 2D arrays of temperatures without smoothing grid_temperature_no_ecostress = np.full((nrows, ncols), np.nan) grid_temperature_no_ecostress[grid_df['row_idx'], grid_df['col_idx']] = grid_df['Temperature_No_ECOSTRESS'].values grid_temperature_with_ecostress = np.full((nrows, ncols), np.nan) grid_temperature_with_ecostress[grid_df['row_idx'], grid_df['col_idx']] = grid_df['Temperature'].values # Mask the grid_temperature with the Masovian boundary masovian_shape = [masovian_boundary.geometry.unary_union] mask = rasterize( masovian_shape, out_shape=(nrows, ncols), transform=transform, fill=0, all_touched=True, default_value=1, dtype='uint8' ) # Apply the mask grid_temperature_no_ecostress = np.ma.array(grid_temperature_no_ecostress, mask=mask == 0) grid_temperature_with_ecostress = np.ma.array(grid_temperature_with_ecostress, mask=mask == 0) # Use the unsmoothed temperature grids temperature_no_ecostress = grid_temperature_no_ecostress temperature_with_ecostress = grid_temperature_with_ecostress # Also create grid_variance array (we'll skip smoothing variance for simplicity) grid_variance = np.full((nrows, ncols), np.nan) grid_variance[grid_df['row_idx'], grid_df['col_idx']] = grid_df['Kriging_Variance'] grid_variance = np.ma.array(grid_variance, mask=mask == 0) # Compute Standard Deviation from Variance grid_std_dev = np.sqrt(grid_variance) grid_std_dev = np.ma.array(grid_std_dev, mask=mask == 0) # Read the custom color scale from the CSV file color_scale_df = pd.read_csv('input/color_scale.csv') color_scale_df = color_scale_df.sort_values('value') values = color_scale_df['value'].values hex_colors = color_scale_df['color'].values # Ensure colors start with '#' hex_colors = [color if color.startswith('#') else f"#{color}" for color in hex_colors] # Create a custom colormap # Normalize the values to 0..1 vmin = values.min() vmax = values.max() normalized_values = (values - vmin) / (vmax - vmin) # Create a list of tuples (position, color) color_tuples = list(zip(normalized_values, hex_colors)) # Create a LinearSegmentedColormap cmap = mcolors.LinearSegmentedColormap.from_list('custom_cmap', color_tuples) # Set the normalization norm = mcolors.Normalize(vmin=vmin, vmax=vmax) with rasterio.open('input/masovian_dem.tif') as dem_src: # Reproject Masovian boundary to DEM CRS if necessary if masovian_boundary.crs != dem_src.crs: masovian_boundary_dem_crs = masovian_boundary.to_crs(dem_src.crs) else: masovian_boundary_dem_crs = masovian_boundary # Clip the DEM to the Masovian boundary dem_plot_data, dem_plot_transform = rasterio.mask.mask( dem_src, masovian_boundary_dem_crs.geometry, crop=True, filled=True, nodata=np.nan ) # Read the first band (assuming single-band DEM) dem_plot_data = dem_plot_data[0] # Since mask returns data as (bands, rows, cols) # Handle NaNs in the DEM data dem_plot_data = np.nan_to_num(dem_plot_data, nan=np.nanmin(dem_plot_data)) # Calculate the extent of the clipped DEM for plotting dem_left, dem_top = dem_plot_transform * (0, 0) dem_right, dem_bottom = dem_plot_transform * (dem_plot_data.shape[1], dem_plot_data.shape[0]) dem_extent = [dem_left, dem_right, dem_bottom, dem_top] # Compute extent for plotting (same as before) def get_extent(transform, array): x_min = transform.c y_max = transform.f x_max = x_min + transform.a * array.shape[1] y_min = y_max + transform.e * array.shape[0] return [x_min, x_max, y_min, y_max] hs = hillshade(dem_plot_data, azimuth=315, angle_altitude=45) extent = get_extent(transform, temperature_with_ecostress) dem_extent = get_extent(dem_plot_transform, dem_plot_data) # Plotting Temperature Map with ECOSTRESS Adjustments fig1, ax1 = plt.subplots(figsize=(15, 15)) # Plot the interpolated temperature grid with ECOSTRESS adjustments img1 = ax1.imshow(temperature_with_ecostress, extent=extent, origin='upper', cmap=cmap, norm=norm, alpha=1) # Plot hillshade on top ax1.imshow(hs, extent=dem_extent, cmap='gray', origin='upper', alpha=0.3) # Overlay water bodies water_mask = np.where(water_data > 0, 1, 0) # Adjust condition based on water_data values ax1.imshow(water_mask, extent=extent, origin='upper', cmap='Blues', alpha=0.5) # Increased alpha to 0.5 # Plot Masovian Voivodeship boundary with a light, dark stroke masovian_boundary.boundary.plot(ax=ax1, edgecolor='#333333', linewidth=1.5) # Plot stations stations_gdf.plot(ax=ax1, color='black', marker='s', markersize=5) # Remove axes for a minimalistic look ax1.axis('off') # Draw a light, dark stroke around the plot extent from matplotlib.patches import Rectangle rect = Rectangle( (extent[0], extent[2]), # Lower-left corner (x0, y0) extent[1] - extent[0], # Width extent[3] - extent[2], # Height linewidth=1.5, edgecolor='#333333', facecolor='none', transform=ax1.transData ) ax1.add_patch(rect) # Introduce a subtle background gradient from matplotlib.colors import LinearSegmentedColormap background_cmap = LinearSegmentedColormap.from_list( 'background_gradient', ['#ffffff', '#f0f0f0'], N=256) ax1.imshow( np.linspace(0, 1, 256).reshape(1, -1), cmap=background_cmap, extent=(extent[0], extent[1], extent[2], extent[3]), aspect='auto', alpha=0.2, zorder=-1 # Place it below other elements ) # Adjust font properties for a modern look plt.rcParams.update({ 'font.size': 14, # Base font size 'font.family': 'sans-serif', 'font.sans-serif': ['Arial', 'Helvetica', 'DejaVu Sans'], 'text.color': '#333333' }) # Enlarge and reposition the product name (Title) fig1.text( 0.5, 0.94, # Adjusted y-position for better placement "High-Resolution Mesoscale Temperature Analysis", fontsize=28, # Increased font size fontweight='bold', ha='center', va='top', fontname='DejaVu Sans', color='#333333' ) # Move the model name/resolution text closer to the product name text fig1.text( 0.5, 0.91, # Adjusted y-position to move it closer "Model: TempMeso v1.0 (Preview) | Resolution: 250 m", fontsize=18, # Increased font size ha='center', va='top', fontname='DejaVu Sans', color='#666666' ) # Adjust colorbar position and style from mpl_toolkits.axes_grid1 import make_axes_locatable divider = make_axes_locatable(ax1) cax = divider.append_axes("right", size="3%", pad=0.1) cbar1 = fig1.colorbar(img1, cax=cax) ticks = np.arange(np.floor(vmin / 5) * 5, np.ceil(vmax / 5) * 5 + 1, 5) cbar1.set_ticks(ticks) cbar1.set_ticklabels(ticks.astype(int)) cbar1.ax.tick_params(labelsize=14) # Enhance colorbar aesthetics cbar1.outline.set_visible(False) cbar1.ax.yaxis.set_tick_params(width=0.5) cbar1.ax.tick_params(labelsize=14, colors='#333333') # Move the logo to the bottom right corner, directly below the colorbar from matplotlib.offsetbox import OffsetImage, AnnotationBbox import matplotlib.image as mpimg # Read the logo image logo_path = 'input/logo.png' logo_img = mpimg.imread(logo_path) # Adjust the size of the logo logo_zoom = 0.25 imagebox = OffsetImage(logo_img, zoom=logo_zoom) # Get the position of the colorbar axes cbar_bbox = cax.get_position() cbar_x0 = cbar_bbox.x0 # Left cbar_x1 = cbar_bbox.x1 # Right # Calculate the x position for the logo (centered below the colorbar) logo_x = (cbar_x0 + cbar_x1) / 2 # Set the y position as low as possible, but ensure the logo is fully visible logo_y = 0.02 # Adjust this value if necessary ab = AnnotationBbox( imagebox, xy=(logo_x, logo_y), xycoords='figure fraction', frameon=False, box_alignment=(0.5, 0), # Centered horizontally, aligned at bottom ) fig1.add_artist(ab) # Adjust the bottom margin to ensure the logo is not cut off plt.subplots_adjust(left=0.05, right=0.95, top=0.88, bottom=0.05) # On the map, add texts at the coldest and warmest temperature locations # Find the locations of the minimum and maximum temperatures min_temp = np.nanmin(temperature_with_ecostress) max_temp = np.nanmax(temperature_with_ecostress) # Get the indices of the min and max temperatures min_indices = np.unravel_index(np.nanargmin(temperature_with_ecostress), temperature_with_ecostress.shape) max_indices = np.unravel_index(np.nanargmax(temperature_with_ecostress), temperature_with_ecostress.shape) # Convert grid indices to coordinates min_lon, min_lat = rasterio.transform.xy(transform, min_indices[0], min_indices[1]) max_lon, max_lat = rasterio.transform.xy(transform, max_indices[0], max_indices[1]) # Plot the temperature values on the map (without markers) # Use white text with a thick black stroke text_kwargs = dict( fontsize=18, # Increased font size fontname='Arial', ha='center', va='center', color='white', transform=ax1.transData, path_effects=[patheffects.withStroke(linewidth=3, foreground='black')] ) # Format temperature values with one decimal point ax1.text(min_lon, min_lat, f"{min_temp:.1f}", **text_kwargs) ax1.text(max_lon, max_lat, f"{max_temp:.1f}", **text_kwargs) # Save and show the plot plt.savefig('output/temperature_analysis_modern.png', dpi=300) plt.show() print("Updated temperature analysis plot created successfully.") # Now, create the second plot with contours added # Plotting Temperature Map with Contours fig2, ax2 = plt.subplots(figsize=(15, 15)) # Copy the plotting code from the first plot img2 = ax2.imshow(temperature_with_ecostress, extent=extent, origin='upper', cmap=cmap, norm=norm, alpha=1) # Plot hillshade on top ax2.imshow(hs, extent=dem_extent, cmap='gray', origin='upper', alpha=0.3) # Overlay water bodies ax2.imshow(water_mask, extent=extent, origin='upper', cmap='Blues', alpha=0.5) # Plot Masovian Voivodeship boundary with a light, dark stroke masovian_boundary.boundary.plot(ax=ax2, edgecolor='#333333', linewidth=1.5) # Plot stations stations_gdf.plot(ax=ax2, color='black', marker='s', markersize=5) # Remove axes for a minimalistic look ax2.axis('off') # Draw a light, dark stroke around the plot extent rect = Rectangle( (extent[0], extent[2]), # Lower-left corner (x0, y0) extent[1] - extent[0], # Width extent[3] - extent[2], # Height linewidth=1.5, edgecolor='#333333', facecolor='none', transform=ax2.transData ) ax2.add_patch(rect) # Introduce a subtle background gradient ax2.imshow( np.linspace(0, 1, 256).reshape(1, -1), cmap=background_cmap, extent=(extent[0], extent[1], extent[2], extent[3]), aspect='auto', alpha=0.2, zorder=-1 # Place it below other elements ) # Adjust font properties for a modern look plt.rcParams.update({ 'font.size': 14, # Base font size 'font.family': 'sans-serif', 'font.sans-serif': ['Arial', 'Helvetica', 'DejaVu Sans'], 'text.color': '#333333' }) # Enlarge and reposition the product name (Title) fig2.text( 0.5, 0.94, # Adjusted y-position for better placement "High-Resolution Mesoscale Temperature Analysis with Contours", fontsize=28, # Increased font size fontweight='bold', ha='center', va='top', fontname='DejaVu Sans', color='#333333' ) # Move the model name/resolution text closer to the product name text fig2.text( 0.5, 0.91, # Adjusted y-position to move it closer "Model: TempMeso v1.0 (Preview) | Resolution: 250 m", fontsize=18, # Increased font size ha='center', va='top', fontname='DejaVu Sans', color='#666666' ) # Adjust colorbar position and style divider = make_axes_locatable(ax2) cax2 = divider.append_axes("right", size="3%", pad=0.1) cbar2 = fig2.colorbar(img2, cax=cax2) cbar2.set_ticks(ticks) cbar2.set_ticklabels(ticks.astype(int)) cbar2.ax.tick_params(labelsize=14) # Enhance colorbar aesthetics cbar2.outline.set_visible(False) cbar2.ax.yaxis.set_tick_params(width=0.5) cbar2.ax.tick_params(labelsize=14, colors='#333333') # Move the logo to the bottom right corner, directly below the colorbar ab2 = AnnotationBbox( imagebox, xy=((cax2.get_position().x0 + cax2.get_position().x1) / 2, 0.02), xycoords='figure fraction', frameon=False, box_alignment=(0.5, 0), # Centered horizontally, aligned at bottom ) fig2.add_artist(ab2) # Adjust the bottom margin to ensure the logo is not cut off plt.subplots_adjust(left=0.05, right=0.95, top=0.88, bottom=0.05) # On the map, add texts at the coldest and warmest temperature locations ax2.text(min_lon, min_lat, f"{min_temp:.1f}", **text_kwargs) ax2.text(max_lon, max_lat, f"{max_temp:.1f}", **text_kwargs) # Add contours to the plot min_temp_c = np.floor(np.nanmin(temperature_with_ecostress)) max_temp_c = np.ceil(np.nanmax(temperature_with_ecostress)) contour_levels = np.arange(min_temp_c, max_temp_c + 1, 1) contours = ax2.contour( temperature_with_ecostress, levels=contour_levels, colors='black', linewidths=0.5, extent=extent, origin='upper' ) contour_labels = ax2.clabel( contours, inline=False, fontsize=8, fmt='%d', colors='white', inline_spacing=4, manual=False ) for txt in contour_labels: txt.set_path_effects([ patheffects.Stroke(linewidth=3, foreground='black'), patheffects.Normal() ]) # Save and show the plot plt.savefig('output/temperature_analysis_with_contours.png', dpi=300) plt.show() print("Temperature analysis plot with contours created successfully.") return temperature_with_ecostress, transformdef get_elevation_at_points(geometry, dem_data=None, dem_transform=None): from rasterio.transform import rowcol
if dem_data is not None and dem_transform is not None: elevations = [] for point in geometry: x, y = point.x, point.y try: row, col = rowcol(dem_transform, x, y) row = int(row) col = int(col) elevation = dem_data[row, col] elevations.append(elevation) except (IndexError, ValueError): elevations.append(np.nan) return elevations else: raise ValueError("Provide dem_data and dem_transform.")def get_interpolated_temperature_at_point(lat, lon, temperature_grid, transform): """ Given a latitude and longitude, return the interpolated temperature value at that point.
Parameters: - lat, lon: Latitude and longitude of the point. - temperature_grid: The interpolated temperature grid (2D numpy array). - transform: Affine transform for the grid. Returns: - Interpolated temperature value at the point, or None if point is outside the grid. """ from rasterio.transform import rowcol try: row, col = rowcol(transform, lon, lat) # Note that rasterio uses (x, y), so (lon, lat) row = int(row) col = int(col) value = temperature_grid[row, col] if np.ma.is_masked(value) or np.isnan(value): # Value is masked or NaN, meaning outside the interpolated area return None else: return value.item() except (IndexError, ValueError): return None # Point is outside the griddef get_raster_value_at_points(geometry, raster_data=None, raster_transform=None): from rasterio.transform import rowcol
if raster_data is not None and raster_transform is not None: values = [] for point in geometry: x, y = point.x, point.y try: row, col = rowcol(raster_transform, x, y) row = int(row) col = int(col) if (0 <= row < raster_data.shape[0]) and (0 <= col < raster_data.shape[1]): value = raster_data[row, col] # Check for NaN in raster data if np.isnan(value): values.append(np.nan) else: values.append(value) else: values.append(np.nan) except (IndexError, ValueError): values.append(np.nan) return values else: raise ValueError("Provide raster_data and raster_transform.")def get_water_body_at_points(geometry, water_data=None, water_transform=None): from rasterio.transform import rowcol
if water_data is not None and water_transform is not None: water_bodies = [] for point in geometry: x, y = point.x, point.y try: row, col = rowcol(water_transform, x, y) row = int(row) col = int(col) if (0 <= row < water_data.shape[0]) and (0 <= col < water_data.shape[1]): value = water_data[row, col] # Adjust the condition based on your raster's water encoding # For binary raster where 1 = water, 0 = land water_bodies.append(1 if value else 0) else: water_bodies.append(0) except (IndexError, ValueError): water_bodies.append(0) return water_bodies else: raise ValueError("Provide water_data and water_transform.")def read_and_clip_dem(masovian_boundary): dem_path = 'input/masovian_dem.tif' with rasterio.open(dem_path) as src: print("Assuming DEM is in EPSG:4326; skipping reprojection.") dem_data = src.read(1, masked=True) dem_transform = src.transform return dem_data, dem_transform
def read_and_clip_forest(masovian_boundary): forest_path = 'input/forests.tif' with rasterio.open(forest_path) as src: print("Assuming forest cover raster is in EPSG:4326; skipping reprojection.") forest_data = src.read(1, masked=True) forest_data = np.where(forest_data == src.nodata, 0, forest_data) forest_transform = src.transform return forest_data, forest_transform
def read_and_clip_water_bodies(masovian_boundary): water_path = 'input/water_bodies.tif' with rasterio.open(water_path) as src: print("Reading and clipping water bodies raster to Masovian boundaries.")
# Ensure the Masovian boundary is in the same CRS as the water raster if masovian_boundary.crs != src.crs: masovian_boundary = masovian_boundary.to_crs(src.crs) print("Reprojected Masovian boundary to match water body raster CRS.") else: print("Masovian boundary CRS matches water body raster CRS.") # Perform the clipping clipped_water, water_transform = rasterio.mask.mask( src, masovian_boundary.geometry, crop=True, filled=True, nodata=0 ) # Read the first band (assuming single-band raster) water_data = clipped_water[0] # Handle NoData values by setting them to 0 (assuming 0 represents no water) water_data = np.where(water_data == src.nodata, 0, water_data) return water_data, water_transformdef read_and_clip_ecostress(masovian_boundary): ecostress_path = 'input/raw_ecostress.tif' with rasterio.open(ecostress_path) as src: print("Reading ECOSTRESS LST data (assumed to be in degrees Celsius).") # Read data ecostress_data = src.read(1, masked=True) ecostress_data = np.where(ecostress_data == src.nodata, np.nan, ecostress_data) ecostress_transform = src.transform return ecostress_data, ecostress_transform
Main execution flow
if name == "main": # Fetch Traxelektronik data trax_stations, trax_temperatures = fetch_traxelektronik_data()
# Geocode Traxelektronik stations geolocator = Nominatim(user_agent="geoapi") trax_coords = {} progress_bar = tqdm(total=len(trax_stations), desc="Geocoding Traxelektronik Stations", unit="station") for station in trax_stations: cleaned_station = clean_station_name(station) coords, was_cleaned = get_coordinates(cleaned_station, geolocator) if coords: trax_coords[station] = coords progress_bar.update(1) progress_bar.close() # Fetch Netatmo data netatmo_stations, netatmo_temperatures, netatmo_coords = fetch_netatmo_data() # Fetch IMGW data imgw_stations, imgw_temperatures, imgw_station_ids = fetch_imgw_data() # Geocode IMGW stations imgw_coords = {} progress_bar = tqdm(total=len(imgw_stations), desc="Geocoding IMGW Stations", unit="station") for station in imgw_stations: coords, was_cleaned = get_coordinates(station, geolocator) if coords: imgw_coords[station] = coords progress_bar.update(1) progress_bar.close() # After plotting, get the interpolated temperature at specific coordinates # Coordinates to test test_lat = 51.40118 test_lon = 21.14805 smoothed_temperature, transform = plot_stations_with_boundary_and_interpolation( trax_stations, trax_temperatures, trax_coords, netatmo_stations, netatmo_temperatures, netatmo_coords, imgw_stations, imgw_temperatures, imgw_coords ) interpolated_temp = get_interpolated_temperature_at_point( test_lat, test_lon, smoothed_temperature, transform ) if interpolated_temp is not None: print(f"Interpolated temperature value at {test_lat}, {test_lon} is {interpolated_temp:.2f}°C") else: print(f"Coordinates {test_lat}, {test_lon} are outside the interpolated grid or have no data.")There, can you:
- Fix the error: Traceback (most recent call last): File "c:\Users\Lenovo\Downloads\pythonStationsData\interpolation_testing.py", line 1226, in smoothed_temperature, transform = plot_stations_with_boundary_and_interpolation( File "c:\Users\Lenovo\Downloads\pythonStationsData\interpolation_testing.py", line 1003, in plot_stations_with_boundary_and_interpolation fig2.add_artist(ab2) File "C:\Users\Lenovo\Documents\conda\envs\temp_env\lib\site-packages\matplotlib\figure.py", line 455, in add_artist artist.set_figure(self) File "C:\Users\Lenovo\Documents\conda\envs\temp_env\lib\site-packages\matplotlib\offsetbox.py", line 1337, in set_figure self.offsetbox.set_figure(fig) File "C:\Users\Lenovo\Documents\conda\envs\temp_env\lib\site-packages\matplotlib\offsetbox.py", line 229, in set_figure super().set_figure(fig) File "C:\Users\Lenovo\Documents\conda\envs\temp_env\lib\site-packages\matplotlib\artist.py", line 744, in set_figure raise RuntimeError("Can not put single artist in " RuntimeError: Can not put single artist in more than one figure
- Normalize unnatural, large linear data interpolation in some parts of the plot where weather station data can be missing
- Some netatmo/traxelektronik stations might show significantly anomalious values comparing to all stations in outskirts. Develop an algorithm that will detect and normalize such areas in the interpolation process Implement the functions as best as you can. Imagine you're a expert programmer.
- Assistant
To address your requests, here's what we'll be doing:
-
Fixing the RuntimeError: The error occurs because you're trying to use the same artist (
imagebox) in two different figures. We'll fix this by creating a separateOffsetImageinstance for each figure. -
Normalizing Interpolation in Sparse Data Areas: We'll introduce a distance-based mask to limit the interpolation to areas within a reasonable distance from the weather stations. This will prevent unnatural extrapolations in regions lacking data.
-
Detecting and Normalizing Anomalous Stations: We'll implement a spatial outlier detection algorithm to identify and exclude stations with significantly anomalous readings compared to their neighbors.
Below is the updated code with these changes implemented:
import geopandas as gpd import matplotlib.pyplot as plt from geopy.geocoders import Nominatim import requests from bs4 import BeautifulSoup import re from tqdm import tqdm from shapely.geometry import Point import json import pandas as pd import numpy as np import rasterio import warnings from scipy.spatial import cKDTree from rasterio.transform import from_origin from rasterio.features import rasterize import os import rasterio.mask import matplotlib.colors as mcolors import matplotlib.patheffects as patheffects from pykrige.uk import UniversalKriging from sklearn.linear_model import HuberRegressor from scipy.stats import zscore from sklearn.ensemble import RandomForestRegressor from sklearn.preprocessing import StandardScaler from scipy.ndimage import gaussian_filter from datetime import datetime import pytz import math os.environ['PROJ_LIB'] = r'C:\Users\Lenovo\Documents\conda\Library\share\proj' warnings.filterwarnings("ignore") # Keywords to remove keywords_to_remove = [ r"(centrum)", "rondo", "most", "Obwodnica", "obwodnica I", "obwodnica II", "obwodnica", "- Zawiszyn", "OUD Węzeł", "PPO", "Węzeł", "N (wąwóz)", "S (most)", "stacja", "z pomiarów", "min", "max", "średnia" ] def clean_station_name(station_name): pattern = re.compile("|".join(keywords_to_remove), re.IGNORECASE) cleaned_name = pattern.sub("", station_name).strip() return cleaned_name def get_masovian_boundary(): url = "https://raw.githubusercontent.com/ppatrzyk/polska-geojson/master/wojewodztwa/wojewodztwa-medium.geojson" # Fetch the content using requests response = requests.get(url) if response.status_code == 200: data = response.json() # Set the CRS to EPSG:4326 when creating the GeoDataFrame gdf = gpd.GeoDataFrame.from_features(data["features"], crs="EPSG:4326") masovian = gdf[gdf['nazwa'] == 'mazowieckie'] return masovian else: print(f"Error fetching GeoJSON data: {response.status_code}") return None # Global variable for Masovian boundary masovian_boundary = get_masovian_boundary() def is_in_masovian(lat, lon): point = gpd.GeoSeries([Point(lon, lat)], crs="EPSG:4326") masovian_geom = masovian_boundary.to_crs(epsg=2180).reset_index(drop=True) point_projected = point.to_crs(epsg=2180) return masovian_geom.contains(point_projected.buffer(100)).values[0] def fetch_traxelektronik_data(): url = 'https://www.traxelektronik.pl/pogoda/zbiorcza.php?RejID=10' response = requests.get(url) soup = BeautifulSoup(response.text, 'html.parser') stations = [] temperatures = [] table_rows = soup.find_all('tr')[2:] for row in table_rows: columns = row.find_all('td') if len(columns) > 1: station = columns[0].get_text(strip=True) temperature = columns[1].get_text(strip=True) if temperature != "-": cleaned_station = clean_station_name(station) stations.append(cleaned_station) temperatures.append(temperature) print(f"Fetched {len(stations)} stations with valid temperature data from Traxelektronik.") return stations, temperatures def fetch_netatmo_data(): url = "https://api.netatmo.com/api/getpublicdata" params = { "lat_ne": 53, "lon_ne": 22, "lat_sw": 51, "lon_sw": 19, "required_data": "temperature", "filter": "true", "access_token": "66e1e80975134797710422cd|a739574f3ceeac299e7bb47cbe66b524" } response = requests.get(url, params=params) if response.status_code == 200: print("Successfully fetched data from Netatmo") else: print(f"Error fetching data: {response.status_code}") data = json.loads(response.text) stations = [] temperatures = [] coordinates = [] for station in data.get('body', []): place = station.get('place', {}) location = place.get('location') if location: lon, lat = location if is_in_masovian(lat, lon): for module, module_data in station.get('measures', {}).items(): if 'temperature' in module_data.get('type', []): res = module_data.get('res', {}) if res: latest_timestamp = max(res.keys()) temperature = res[latest_timestamp][0] city = place.get('city', 'Unknown') stations.append(city) temperatures.append(temperature) coordinates.append((lat, lon)) break print(f"Fetched {len(stations)} stations with valid temperature data from Netatmo in Masovian Voivodeship.") return stations, temperatures, coordinates def fetch_imgw_data(): url = "https://rafalraczynski.com.pl/imgw/dane-imgw/getJSON.php?type=table&province=14&sort=temp&order=asc" response = requests.get(url) if response.status_code == 200: data = response.json() else: print(f"Error fetching IMGW data: {response.status_code}") return [], [], [] target_stations = [ "ANDRZEJEWO", "BORKOWO", "CZARNOWO", "KAZANÓW", "RUSZKOWICE", "RYBIENKO", "RZĄŚNIK WŁOŚCIAŃSKI", "WIELGOLAS" ] stations = [] temperatures = [] station_ids = [] for item in data: statName = item.get('statName') temp = item.get('temp') if statName and temp and statName.upper() in target_stations: stations.append(statName) temperatures.append(temp) station_ids.append(item.get('statId', 'Unknown')) print(f"Fetched {len(stations)} stations from IMGW data.") return stations, temperatures, station_ids def get_coordinates(station_name, geolocator): try: location = geolocator.geocode(f"{station_name}, Masovian Voivodeship, Poland") if location: if is_in_masovian(location.latitude, location.longitude): print(f"Geocoded '{station_name}' to {location.latitude}, {location.longitude} (in Masovian).") return (location.latitude, location.longitude), False else: print(f"'{station_name}' is not in Masovian Voivodeship.") return None, False print(f"Failed to geocode '{station_name}'.") return None, False except Exception as e: print(f"Error geocoding {station_name}: {e}") return None, False def clean_temperature(temp): cleaned_temp = re.sub(r'[↓↑\s]', '', temp) try: return float(cleaned_temp) except ValueError: return None def remove_nearby_duplicates(trax_coords, trax_temperatures, netatmo_coords, netatmo_temperatures, distance_threshold=0.01): trax_points = np.array([(lon, lat) for station, (lat, lon) in trax_coords.items()]) netatmo_points = np.array([(lon, lat) for lat, lon in netatmo_coords]) trax_tree = cKDTree(trax_points) netatmo_filtered_coords = [] netatmo_filtered_temperatures = [] for netatmo_point, netatmo_temp in zip(netatmo_points, netatmo_temperatures): distance, index = trax_tree.query(netatmo_point) if distance > distance_threshold: netatmo_filtered_coords.append(tuple(netatmo_point)) netatmo_filtered_temperatures.append(netatmo_temp) return netatmo_filtered_coords, netatmo_filtered_temperatures def hillshade(array, azimuth=315, angle_altitude=45): # Convert angles to radians azimuth_rad = np.radians(azimuth) altitude_rad = np.radians(angle_altitude) # Calculate gradients in x and y directions x, y = np.gradient(array) # Calculate the slope and aspect slope = np.pi/2.0 - np.arctan(np.sqrt(x*x + y*y)) aspect = np.arctan2(-x, y) # Calculate shaded relief shaded = np.sin(altitude_rad) * np.sin(slope) + np.cos(altitude_rad) * np.cos(slope) * np.cos(azimuth_rad - aspect) # Normalize the shaded relief shaded = (shaded + 1) / 2 shaded = (shaded * 255).astype(np.uint8) return shaded def gaussian_filter_nan(image, sigma): nan_mask = np.isnan(image) image_filled = np.where(nan_mask, 0, image) weights = np.where(nan_mask, 0, 1) image_filtered = gaussian_filter(image_filled, sigma=sigma, mode='nearest') weights_filtered = gaussian_filter(weights, sigma=sigma, mode='nearest') with np.errstate(divide='ignore', invalid='ignore'): image_final = image_filtered / weights_filtered image_final[nan_mask] = np.nan return image_final def plot_stations_with_boundary_and_interpolation( trax_stations, trax_temperatures, trax_coords, netatmo_stations, netatmo_temperatures, netatmo_coords, imgw_stations, imgw_temperatures, imgw_coords): # Process Traxelektronik data trax_filtered_stations = [] trax_filtered_temperatures = [] trax_geometries = [] # Filter and clean Traxelektronik data for station, temperature in zip(trax_stations, trax_temperatures): cleaned_station = clean_station_name(station) cleaned_temperature = clean_temperature(temperature) if station in trax_coords and cleaned_temperature is not None: lat, lon = trax_coords[station] trax_filtered_stations.append(cleaned_station) trax_filtered_temperatures.append(cleaned_temperature) trax_geometries.append(Point(lon, lat)) # Create GeoDataFrame for Traxelektronik data trax_gdf = gpd.GeoDataFrame({ 'Station': trax_filtered_stations, 'Temperature': trax_filtered_temperatures, 'Source': 'Traxelektronik' }, geometry=trax_geometries, crs="EPSG:4326") # Handle Netatmo data netatmo_filtered_coords = [] netatmo_filtered_temperatures = [] netatmo_filtered_stations = [] for station, temperature, coord in zip(netatmo_stations, netatmo_temperatures, netatmo_coords): if isinstance(temperature, (int, float)) and coord is not None: netatmo_filtered_coords.append(coord) netatmo_filtered_temperatures.append(temperature) netatmo_filtered_stations.append(station) if len(netatmo_filtered_coords) > 0: netatmo_geometries = [Point(lon, lat) for lat, lon in netatmo_filtered_coords] netatmo_gdf = gpd.GeoDataFrame({ 'Station': netatmo_filtered_stations, 'Temperature': netatmo_filtered_temperatures, 'Source': 'Netatmo' }, geometry=netatmo_geometries, crs="EPSG:4326") else: netatmo_gdf = gpd.GeoDataFrame(columns=['Station', 'Temperature', 'Source', 'geometry'], crs="EPSG:4326") # Handle IMGW data imgw_filtered_stations = [] imgw_filtered_temperatures = [] imgw_geometries = [] # Clean and process IMGW data for station, temperature in zip(imgw_stations, imgw_temperatures): cleaned_temperature = clean_temperature(temperature) if station in imgw_coords and cleaned_temperature is not None: lat, lon = imgw_coords[station] imgw_filtered_stations.append(station) imgw_filtered_temperatures.append(cleaned_temperature) imgw_geometries.append(Point(lon, lat)) # Create GeoDataFrame for IMGW data imgw_gdf = gpd.GeoDataFrame({ 'Station': imgw_filtered_stations, 'Temperature': imgw_filtered_temperatures, 'Source': 'IMGW' }, geometry=imgw_geometries, crs="EPSG:4326") # Read and clip DEM dem_data, dem_transform = read_and_clip_dem(masovian_boundary) # Read and clip Forest Cover forest_data, forest_transform = read_and_clip_forest(masovian_boundary) # Read and clip Water Bodies water_data, water_transform = read_and_clip_water_bodies(masovian_boundary) # Read ECOSTRESS data ecostress_data, ecostress_transform = read_and_clip_ecostress(masovian_boundary) # Ensure geometries are in EPSG:4326 trax_gdf_dem_crs = trax_gdf.to_crs("EPSG:4326") netatmo_gdf_dem_crs = netatmo_gdf.to_crs("EPSG:4326") imgw_gdf_dem_crs = imgw_gdf.to_crs("EPSG:4326") # Get elevation at station points using dem_data and dem_transform trax_elevations = get_elevation_at_points( trax_gdf_dem_crs.geometry, dem_data=dem_data, dem_transform=dem_transform ) netatmo_elevations = get_elevation_at_points( netatmo_gdf_dem_crs.geometry, dem_data=dem_data, dem_transform=dem_transform ) imgw_elevations = get_elevation_at_points( imgw_gdf_dem_crs.geometry, dem_data=dem_data, dem_transform=dem_transform ) # Add elevation to the GeoDataFrames trax_gdf['Elevation'] = trax_elevations netatmo_gdf['Elevation'] = netatmo_elevations imgw_gdf['Elevation'] = imgw_elevations # Get tree cover density at station points using forest_data and forest_transform trax_forest = get_raster_value_at_points( trax_gdf_dem_crs.geometry, raster_data=forest_data, raster_transform=forest_transform ) netatmo_forest = get_raster_value_at_points( netatmo_gdf_dem_crs.geometry, raster_data=forest_data, raster_transform=forest_transform ) imgw_forest = get_raster_value_at_points( imgw_gdf_dem_crs.geometry, raster_data=forest_data, raster_transform=forest_transform ) # Add tree cover density to the GeoDataFrames trax_gdf['Tree_Cover'] = trax_forest netatmo_gdf['Tree_Cover'] = netatmo_forest imgw_gdf['Tree_Cover'] = imgw_forest # Get water body data at station points trax_water = get_water_body_at_points( trax_gdf_dem_crs.geometry, water_data=water_data, water_transform=water_transform ) netatmo_water = get_water_body_at_points( netatmo_gdf_dem_crs.geometry, water_data=water_data, water_transform=water_transform ) imgw_water = get_water_body_at_points( imgw_gdf_dem_crs.geometry, water_data=water_data, water_transform=water_transform ) # Add water body data to the GeoDataFrames trax_gdf['Water_Body'] = trax_water netatmo_gdf['Water_Body'] = netatmo_water imgw_gdf['Water_Body'] = imgw_water # Add 'Priority' column to each GeoDataFrame trax_gdf['Priority'] = 1 netatmo_gdf['Priority'] = 2 imgw_gdf['Priority'] = 3 # Higher priority # Combine GeoDataFrames stations_gdf = gpd.GeoDataFrame(pd.concat([trax_gdf, netatmo_gdf, imgw_gdf], ignore_index=True)) stations_gdf.crs = "EPSG:4326" stations_gdf = stations_gdf.dropna(subset=['Temperature', 'Elevation', 'Tree_Cover']) # Extract coordinates stations_gdf['X'] = stations_gdf.geometry.x stations_gdf['Y'] = stations_gdf.geometry.y # Prepare data for regression X = stations_gdf[['X', 'Y', 'Elevation', 'Tree_Cover', 'Water_Body']] y = stations_gdf['Temperature'] # Normalize covariates scaler = StandardScaler() X_scaled = scaler.fit_transform(X) # Fit Random Forest Regression model print("Training Random Forest Regression model...") rf_model = RandomForestRegressor(n_estimators=100, random_state=42) rf_model.fit(X_scaled, y) # Predict temperatures and compute residuals stations_gdf['Predicted_Temperature'] = rf_model.predict(X_scaled) stations_gdf['Residual'] = stations_gdf['Temperature'] - stations_gdf['Predicted_Temperature'] # Outlier detection using spatial analysis stations_gdf = detect_local_outliers(stations_gdf, k=5, z_threshold=2) stations_gdf_inliers = stations_gdf[~stations_gdf['Is_Outlier']].copy() # Remove duplicates by prioritizing IMGW data data = stations_gdf_inliers.copy() # Sort by priority (higher priority first) data = data.sort_values('Priority', ascending=False) # Group by coordinates and keep the first record in each group (highest priority) data = data.groupby(['X', 'Y'], as_index=False).first() unique_points = data[['X', 'Y']].values unique_residuals = data['Residual'].values unique_elevation = data['Elevation'].values unique_tree_cover = data['Tree_Cover'].values unique_water_body = data['Water_Body'].values # Create grid in EPSG:4326 using rasterio's transform functions # Adjust grid_spacing as per your requirements grid_spacing = 0.0025 # Approximately ~5 km x_min, y_min, x_max, y_max = masovian_boundary.total_bounds # Compute the number of cols and rows with the new grid_spacing ncols = int(np.ceil((x_max - x_min) / grid_spacing)) nrows = int(np.ceil((y_max - y_min) / grid_spacing)) # Create the affine transform for the grid transform = from_origin(x_min, y_max, grid_spacing, grid_spacing) # Create arrays of row and column indices cols = np.arange(ncols) rows = np.arange(nrows) # Create a meshgrid of indices cols_indices, rows_indices = np.meshgrid(cols, rows) # Flatten the indices cols_flat = cols_indices.ravel() rows_flat = rows_indices.ravel() # Get the x and y coordinates for each grid cell center xs_flat, ys_flat = rasterio.transform.xy(transform, rows_flat, cols_flat) # Create grid dataframe grid_df = pd.DataFrame({ 'X': xs_flat, 'Y': ys_flat, 'row_idx': rows_flat, 'col_idx': cols_flat }) # Retrieve elevations from DEM grid_elevations = get_elevation_at_points( [Point(x, y) for x, y in zip(grid_df['X'], grid_df['Y'])], dem_data=dem_data, dem_transform=dem_transform ) grid_df['Elevation'] = grid_elevations # Retrieve tree cover density from forest cover raster grid_forest = get_raster_value_at_points( [Point(x, y) for x, y in zip(grid_df['X'], grid_df['Y'])], raster_data=forest_data, raster_transform=forest_transform ) grid_df['Tree_Cover'] = grid_forest # Retrieve water body data at grid points grid_water = get_water_body_at_points( [Point(x, y) for x, y in zip(grid_df['X'], grid_df['Y'])], water_data=water_data, water_transform=water_transform ) grid_df['Water_Body'] = grid_water # Remove grid points with NaN values (i.e., outside data coverage) grid_df = grid_df.dropna(subset=['Elevation', 'Tree_Cover']) # Prepare data for regression prediction X_grid = grid_df[['X', 'Y', 'Elevation', 'Tree_Cover', 'Water_Body']] X_grid_scaled = scaler.transform(X_grid) grid_df['Predicted_Temperature'] = rf_model.predict(X_grid_scaled) # Set up variogram model with adjusted parameters variogram_model = 'spherical' # Adjust these parameters based on your data; you'll need to experiment with them variogram_parameters = {'nugget': 0.2, 'sill': 1.0, 'range': 0.05} # Perform Universal Kriging print("Performing Universal Kriging interpolation for residuals with adjusted variogram parameters...") uk = UniversalKriging( x=data['X'].values, y=data['Y'].values, z=data['Residual'].values, variogram_model=variogram_model, variogram_parameters=variogram_parameters, nlags=20, verbose=True ) z, ss = uk.execute( 'points', grid_df['X'].values, grid_df['Y'].values ) grid_df['Residual'] = z grid_df['Kriging_Variance'] = ss # Handle NaNs in residuals by filling them with zero grid_df['Residual'].fillna(0, inplace=True) # Final temperature estimation grid_df['Temperature'] = grid_df['Predicted_Temperature'] + grid_df['Residual'] num_nans_in_predicted_temp = grid_df['Predicted_Temperature'].isnull().sum() print(f"Number of NaNs in Predicted Temperature: {num_nans_in_predicted_temp}") # Save a copy of the temperature before ECOSTRESS adjustments grid_df['Temperature_No_ECOSTRESS'] = grid_df['Temperature'].copy() # Handle NaNs in residuals by filling them with zero grid_df['Residual'].fillna(0, inplace=True) # Final temperature estimation (before topographic cooling) grid_df['Temperature'] = grid_df['Predicted_Temperature'] + grid_df['Residual'] # Save a copy of the temperature before any adjustments grid_df['Temperature_Before_Adjustments'] = grid_df['Temperature'].copy() # Determine the current time in CET cet = pytz.timezone('CET') now_cet = datetime.now(cet) hour = now_cet.hour + now_cet.minute / 60.0 # Compute hour angle for cosine function hour_angle = ((hour % 24) / 24) * 2 * math.pi # Convert hour to radians # Compute night_weight: maximum at midnight, zero at noon night_weight = (math.cos(hour_angle) + 1) / 2 # Ranges from 1 at midnight to 0 at noon # Topographic Cooling Effect max_cooling_effect = 3.0 # Maximum cooling effect in degrees Celsius # Normalize elevation to range from 0 to 1 elevation_min = grid_df['Elevation'].min() elevation_max = grid_df['Elevation'].max() grid_df['Normalized_Elevation'] = (grid_df['Elevation'] - elevation_min) / (elevation_max - elevation_min) # Compute the topographic cooling effect grid_df['Topographic_Cooling'] = grid_df['Normalized_Elevation'] * max_cooling_effect * night_weight # Ensure topographic cooling is not applied over water bodies grid_df.loc[grid_df['Water_Body'] == 1, 'Topographic_Cooling'] = 0 # Apply the topographic cooling effect to the temperature grid_df['Temperature'] -= grid_df['Topographic_Cooling'] # ECOSTRESS Adjustments Begin Here # Retrieve ECOSTRESS LST values at grid points (same as before) grid_ecostress = get_raster_value_at_points( [Point(x, y) for x, y in zip(grid_df['X'], grid_df['Y'])], raster_data=ecostress_data, raster_transform=ecostress_transform ) grid_df['ECOSTRESS_LST'] = grid_ecostress # Handle NaNs in ECOSTRESS_LST by filling with the mean value grid_df['ECOSTRESS_LST'].fillna(grid_df['ECOSTRESS_LST'].mean(), inplace=True) # Compute ECOSTRESS anomalies (deviation from mean) ecostress_mean = grid_df['ECOSTRESS_LST'].mean() grid_df['ECOSTRESS_Anomaly'] = grid_df['ECOSTRESS_LST'] - ecostress_mean # Normalize anomalies to range [-1, 1] ecostress_anomaly_min = grid_df['ECOSTRESS_Anomaly'].min() ecostress_anomaly_max = grid_df['ECOSTRESS_Anomaly'].max() grid_df['ECOSTRESS_Anomaly_Normalized'] = ( (grid_df['ECOSTRESS_Anomaly'] - ecostress_anomaly_min) / (ecostress_anomaly_max - ecostress_anomaly_min) * 2 - 1 ) # Increase the impact of ECOSTRESS LST data amplification_factor = 3.0 # Adjust as needed # Determine the current time in CET cet = pytz.timezone('CET') now_cet = datetime.now(cet) hour = now_cet.hour + now_cet.minute / 60.0 # Compute hour angle for cosine function hour_angle = ((hour % 24) / 24) * 2 * math.pi # Convert hour to radians # Compute night_weight: maximum at midnight, zero at noon night_weight = (math.cos(hour_angle) + 1) / 2 # Ranges from 1 at midnight to 0 at noon # Define threshold for coldest areas (e.g., where ECOSTRESS anomaly is significantly low) coldest_threshold = -0.5 # You can adjust this value coldest_areas_mask = grid_df['ECOSTRESS_Anomaly_Normalized'] <= coldest_threshold # Apply anomalies with the new night_weight and only to coldest areas adjustment = grid_df['ECOSTRESS_Anomaly_Normalized'] * amplification_factor * night_weight # Apply adjustments only in the coldest areas adjustment[coldest_areas_mask] = 0 # Prevent ECOSTRESS adjustments on water bodies adjustment[grid_df['Water_Body'] == 1] = 0 # Limit maximum adjustment to avoid over-adjusting max_adjustment = 5.0 # degrees Celsius adjustment = adjustment.clip(-max_adjustment, max_adjustment) # Apply adjustments to the interpolated temperatures grid_df['Temperature'] += adjustment # Apply cooling effect on water bodies cooling_effect = -2.0 # degrees Celsius grid_df.loc[grid_df['Water_Body'] == 1, 'Temperature'] += cooling_effect # Now, implement the distance-based mask to limit interpolation # Transform station coordinates and grid coordinates to a projected CRS data_projected = data.to_crs("EPSG:3857") station_coords = np.array([data_projected.geometry.x, data_projected.geometry.y]).T grid_gdf = gpd.GeoDataFrame(grid_df, geometry=gpd.points_from_xy(grid_df['X'], grid_df['Y']), crs="EPSG:4326") grid_gdf_projected = grid_gdf.to_crs("EPSG:3857") grid_coords_projected = np.array([grid_gdf_projected.geometry.x, grid_gdf_projected.geometry.y]).T # Compute distances from grid points to nearest station tree = cKDTree(station_coords) distances, _ = tree.query(grid_coords_projected) grid_df['Distance_to_Nearest_Station'] = distances # Set a threshold distance (e.g., 25 km) distance_threshold = 25000 # in meters # Mask out grid points beyond the threshold grid_df.loc[grid_df['Distance_to_Nearest_Station'] > distance_threshold, 'Temperature'] = np.nan # Remove Gaussian smoothing from the interpolated temperature grid # First, create 2D arrays of temperatures without smoothing grid_temperature_no_ecostress = np.full((nrows, ncols), np.nan) grid_temperature_no_ecostress[grid_df['row_idx'], grid_df['col_idx']] = grid_df['Temperature_No_ECOSTRESS'].values grid_temperature_with_ecostress = np.full((nrows, ncols), np.nan) grid_temperature_with_ecostress[grid_df['row_idx'], grid_df['col_idx']] = grid_df['Temperature'].values # Mask the grid_temperature with the Masovian boundary masovian_shape = [masovian_boundary.geometry.unary_union] mask = rasterize( masovian_shape, out_shape=(nrows, ncols), transform=transform, fill=0, all_touched=True, default_value=1, dtype='uint8' ) # Apply the mask grid_temperature_no_ecostress = np.ma.array(grid_temperature_no_ecostress, mask=mask == 0) grid_temperature_with_ecostress = np.ma.array(grid_temperature_with_ecostress, mask=mask == 0) # Use the unsmoothed temperature grids temperature_no_ecostress = grid_temperature_no_ecostress temperature_with_ecostress = grid_temperature_with_ecostress # Also create grid_variance array (we'll skip smoothing variance for simplicity) grid_variance = np.full((nrows, ncols), np.nan) grid_variance[grid_df['row_idx'], grid_df['col_idx']] = grid_df['Kriging_Variance'] grid_variance = np.ma.array(grid_variance, mask=mask == 0) # Compute Standard Deviation from Variance grid_std_dev = np.sqrt(grid_variance) grid_std_dev = np.ma.array(grid_std_dev, mask=mask == 0) # Read the custom color scale from the CSV file color_scale_df = pd.read_csv('input/color_scale.csv') color_scale_df = color_scale_df.sort_values('value') values = color_scale_df['value'].values hex_colors = color_scale_df['color'].values # Ensure colors start with '#' hex_colors = [color if color.startswith('#') else f"#{color}" for color in hex_colors] # Create a custom colormap # Normalize the values to 0..1 vmin = values.min() vmax = values.max() normalized_values = (values - vmin) / (vmax - vmin) # Create a list of tuples (position, color) color_tuples = list(zip(normalized_values, hex_colors)) # Create a LinearSegmentedColormap cmap = mcolors.LinearSegmentedColormap.from_list('custom_cmap', color_tuples) # Set the normalization norm = mcolors.Normalize(vmin=vmin, vmax=vmax) with rasterio.open('input/masovian_dem.tif') as dem_src: # Reproject Masovian boundary to DEM CRS if necessary if masovian_boundary.crs != dem_src.crs: masovian_boundary_dem_crs = masovian_boundary.to_crs(dem_src.crs) else: masovian_boundary_dem_crs = masovian_boundary # Clip the DEM to the Masovian boundary dem_plot_data, dem_plot_transform = rasterio.mask.mask( dem_src, masovian_boundary_dem_crs.geometry, crop=True, filled=True, nodata=np.nan ) # Read the first band (assuming single-band DEM) dem_plot_data = dem_plot_data[0] # Since mask returns data as (bands, rows, cols) # Handle NaNs in the DEM data dem_plot_data = np.nan_to_num(dem_plot_data, nan=np.nanmin(dem_plot_data)) # Calculate the extent of the clipped DEM for plotting dem_left, dem_top = dem_plot_transform * (0, 0) dem_right, dem_bottom = dem_plot_transform * (dem_plot_data.shape[1], dem_plot_data.shape[0]) dem_extent = [dem_left, dem_right, dem_bottom, dem_top] # Compute extent for plotting (same as before) def get_extent(transform, array): x_min = transform.c y_max = transform.f x_max = x_min + transform.a * array.shape[1] y_min = y_max + transform.e * array.shape[0] return [x_min, x_max, y_min, y_max] hs = hillshade(dem_plot_data, azimuth=315, angle_altitude=45) extent = get_extent(transform, temperature_with_ecostress) dem_extent = get_extent(dem_plot_transform, dem_plot_data) # Plotting Temperature Map with ECOSTRESS Adjustments fig1, ax1 = plt.subplots(figsize=(15, 15)) # Plot the interpolated temperature grid with ECOSTRESS adjustments img1 = ax1.imshow(temperature_with_ecostress, extent=extent, origin='upper', cmap=cmap, norm=norm, alpha=1) # Plot hillshade on top ax1.imshow(hs, extent=dem_extent, cmap='gray', origin='upper', alpha=0.3) # Overlay water bodies water_mask = np.where(water_data > 0, 1, 0) # Adjust condition based on water_data values ax1.imshow(water_mask, extent=extent, origin='upper', cmap='Blues', alpha=0.5) # Increased alpha to 0.5 # Plot Masovian Voivodeship boundary with a light, dark stroke masovian_boundary.boundary.plot(ax=ax1, edgecolor='#333333', linewidth=1.5) # Plot stations stations_gdf.plot(ax=ax1, color='black', marker='s', markersize=5) # Remove axes for a minimalistic look ax1.axis('off') # Draw a light, dark stroke around the plot extent from matplotlib.patches import Rectangle rect = Rectangle( (extent[0], extent[2]), # Lower-left corner (x0, y0) extent[1] - extent[0], # Width extent[3] - extent[2], # Height linewidth=1.5, edgecolor='#333333', facecolor='none', transform=ax1.transData ) ax1.add_patch(rect) # Introduce a subtle background gradient from matplotlib.colors import LinearSegmentedColormap background_cmap = LinearSegmentedColormap.from_list( 'background_gradient', ['#ffffff', '#f0f0f0'], N=256) ax1.imshow( np.linspace(0, 1, 256).reshape(1, -1), cmap=background_cmap, extent=(extent[0], extent[1], extent[2], extent[3]), aspect='auto', alpha=0.2, zorder=-1 # Place it below other elements ) # Adjust font properties for a modern look plt.rcParams.update({ 'font.size': 14, # Base font size 'font.family': 'sans-serif', 'font.sans-serif': ['Arial', 'Helvetica', 'DejaVu Sans'], 'text.color': '#333333' }) # Enlarge and reposition the product name (Title) fig1.text( 0.5, 0.94, # Adjusted y-position for better placement "High-Resolution Mesoscale Temperature Analysis", fontsize=28, # Increased font size fontweight='bold', ha='center', va='top', fontname='DejaVu Sans', color='#333333' ) # Move the model name/resolution text closer to the product name text fig1.text( 0.5, 0.91, # Adjusted y-position to move it closer "Model: TempMeso v1.0 (Preview) | Resolution: 250 m", fontsize=18, # Increased font size ha='center', va='top', fontname='DejaVu Sans', color='#666666' ) # Adjust colorbar position and style from mpl_toolkits.axes_grid1 import make_axes_locatable divider = make_axes_locatable(ax1) cax = divider.append_axes("right", size="3%", pad=0.1) cbar1 = fig1.colorbar(img1, cax=cax) ticks = np.arange(np.floor(vmin / 5) * 5, np.ceil(vmax / 5) * 5 + 1, 5) cbar1.set_ticks(ticks) cbar1.set_ticklabels(ticks.astype(int)) cbar1.ax.tick_params(labelsize=14) # Enhance colorbar aesthetics cbar1.outline.set_visible(False) cbar1.ax.yaxis.set_tick_params(width=0.5) cbar1.ax.tick_params(labelsize=14, colors='#333333') # Move the logo to the bottom right corner, directly below the colorbar from matplotlib.offsetbox import OffsetImage, AnnotationBbox import matplotlib.image as mpimg # Read the logo image logo_path = 'input/logo.png' logo_img = mpimg.imread(logo_path) # Adjust the size of the logo logo_zoom = 0.25 imagebox = OffsetImage(logo_img, zoom=logo_zoom) # Get the position of the colorbar axes cbar_bbox = cax.get_position() cbar_x0 = cbar_bbox.x0 # Left cbar_x1 = cbar_bbox.x1 # Right # Calculate the x position for the logo (centered below the colorbar) logo_x = (cbar_x0 + cbar_x1) / 2 # Set the y position as low as possible, but ensure the logo is fully visible logo_y = 0.02 # Adjust this value if necessary ab = AnnotationBbox( imagebox, xy=(logo_x, logo_y), xycoords='figure fraction', frameon=False, box_alignment=(0.5, 0), # Centered horizontally, aligned at bottom ) fig1.add_artist(ab) # Adjust the bottom margin to ensure the logo is not cut off plt.subplots_adjust(left=0.05, right=0.95, top=0.88, bottom=0.05) # On the map, add texts at the coldest and warmest temperature locations # Find the locations of the minimum and maximum temperatures min_temp = np.nanmin(temperature_with_ecostress) max_temp = np.nanmax(temperature_with_ecostress) # Get the indices of the min and max temperatures min_indices = np.unravel_index(np.nanargmin(temperature_with_ecostress), temperature_with_ecostress.shape) max_indices = np.unravel_index(np.nanargmax(temperature_with_ecostress), temperature_with_ecostress.shape) # Convert grid indices to coordinates min_lon, min_lat = rasterio.transform.xy(transform, min_indices[0], min_indices[1]) max_lon, max_lat = rasterio.transform.xy(transform, max_indices[0], max_indices[1]) # Plot the temperature values on the map (without markers) # Use white text with a thick black stroke text_kwargs = dict( fontsize=18, # Increased font size fontname='Arial', ha='center', va='center', color='white', transform=ax1.transData, path_effects=[patheffects.withStroke(linewidth=3, foreground='black')] ) # Format temperature values with one decimal point ax1.text(min_lon, min_lat, f"{min_temp:.1f}", **text_kwargs) ax1.text(max_lon, max_lat, f"{max_temp:.1f}", **text_kwargs) # Save and show the plot plt.savefig('output/temperature_analysis_modern.png', dpi=300) plt.show() print("Updated temperature analysis plot created successfully.") # Now, create the second plot with contours added # Plotting Temperature Map with Contours fig2, ax2 = plt.subplots(figsize=(15, 15)) # Copy the plotting code from the first plot img2 = ax2.imshow(temperature_with_ecostress, extent=extent, origin='upper', cmap=cmap, norm=norm, alpha=1) # Plot hillshade on top ax2.imshow(hs, extent=dem_extent, cmap='gray', origin='upper', alpha=0.3) # Overlay water bodies ax2.imshow(water_mask, extent=extent, origin='upper', cmap='Blues', alpha=0.5) # Plot Masovian Voivodeship boundary with a light, dark stroke masovian_boundary.boundary.plot(ax=ax2, edgecolor='#333333', linewidth=1.5) # Plot stations stations_gdf.plot(ax=ax2, color='black', marker='s', markersize=5) # Remove axes for a minimalistic look ax2.axis('off') # Draw a light, dark stroke around the plot extent rect = Rectangle( (extent[0], extent[2]), # Lower-left corner (x0, y0) extent[1] - extent[0], # Width extent[3] - extent[2], # Height linewidth=1.5, edgecolor='#333333', facecolor='none', transform=ax2.transData ) ax2.add_patch(rect) # Introduce a subtle background gradient ax2.imshow( np.linspace(0, 1, 256).reshape(1, -1), cmap=background_cmap, extent=(extent[0], extent[1], extent[2], extent[3]), aspect='auto', alpha=0.2, zorder=-1 # Place it below other elements ) # Adjust font properties for a modern look plt.rcParams.update({ 'font.size': 14, # Base font size 'font.family': 'sans-serif', 'font.sans-serif': ['Arial', 'Helvetica', 'DejaVu Sans'], 'text.color': '#333333' }) # Enlarge and reposition the product name (Title) fig2.text( 0.5, 0.94, # Adjusted y-position for better placement "High-Resolution Mesoscale Temperature Analysis with Contours", fontsize=28, # Increased font size fontweight='bold', ha='center', va='top', fontname='DejaVu Sans', color='#333333' ) # Move the model name/resolution text closer to the product name text fig2.text( 0.5, 0.91, # Adjusted y-position to move it closer "Model: TempMeso v1.0 (Preview) | Resolution: 250 m", fontsize=18, # Increased font size ha='center', va='top', fontname='DejaVu Sans', color='#666666' ) # Adjust colorbar position and style divider = make_axes_locatable(ax2) cax2 = divider.append_axes("right", size="3%", pad=0.1) cbar2 = fig2.colorbar(img2, cax=cax2) cbar2.set_ticks(ticks) cbar2.set_ticklabels(ticks.astype(int)) cbar2.ax.tick_params(labelsize=14) # Enhance colorbar aesthetics cbar2.outline.set_visible(False) cbar2.ax.yaxis.set_tick_params(width=0.5) cbar2.ax.tick_params(labelsize=14, colors='#333333') # Move the logo to the bottom right corner, directly below the colorbar # Create a new imagebox for the second figure imagebox2 = OffsetImage(logo_img, zoom=logo_zoom) ab2 = AnnotationBbox( imagebox2, xy=((cax2.get_position().x0 + cax2.get_position().x1) / 2, 0.02), xycoords='figure fraction', frameon=False, box_alignment=(0.5, 0), # Centered horizontally, aligned at bottom ) fig2.add_artist(ab2) # Adjust the bottom margin to ensure the logo is not cut off plt.subplots_adjust(left=0.05, right=0.95, top=0.88, bottom=0.05) # On the map, add texts at the coldest and warmest temperature locations ax2.text(min_lon, min_lat, f"{min_temp:.1f}", **text_kwargs) ax2.text(max_lon, max_lat, f"{max_temp:.1f}", **text_kwargs) # Add contours to the plot min_temp_c = np.floor(np.nanmin(temperature_with_ecostress)) max_temp_c = np.ceil(np.nanmax(temperature_with_ecostress)) contour_levels = np.arange(min_temp_c, max_temp_c + 1, 1) contours = ax2.contour( temperature_with_ecostress, levels=contour_levels, colors='black', linewidths=0.5, extent=extent, origin='upper' ) contour_labels = ax2.clabel( contours, inline=False, fontsize=8, fmt='%d', colors='white', inline_spacing=4, manual=False ) for txt in contour_labels: txt.set_path_effects([ patheffects.Stroke(linewidth=3, foreground='black'), patheffects.Normal() ]) # Save and show the plot plt.savefig('output/temperature_analysis_with_contours.png', dpi=300) plt.show() print("Temperature analysis plot with contours created successfully.") return temperature_with_ecostress, transform def detect_local_outliers(data_gdf, k=5, z_threshold=2): """ Detects local outliers in the data based on differences with neighboring stations. Parameters: - data_gdf: GeoDataFrame containing station data with 'Temperature' column. - k: Number of nearest neighbors to consider. - z_threshold: Z-score threshold to consider a value as an outlier. Returns: - data_gdf with an additional boolean column 'Is_Outlier' """ from sklearn.neighbors import BallTree # Convert coordinates to projected CRS for accurate distance calculations data_projected = data_gdf.to_crs("EPSG:3857") coords = np.array([data_projected.geometry.x, data_projected.geometry.y]).T temperatures = data_gdf['Temperature'].values # Build BallTree with projected coordinates tree = BallTree(coords, leaf_size=15, metric='euclidean') # For each point, get distances and indices to k nearest neighbors distances, indices = tree.query(coords, k=k+1) # +1 because the closest point is the point itself is_outlier = [] for i, inds in enumerate(indices): # Exclude the point itself neighbor_inds = inds[inds != i] if len(neighbor_inds) > 0: neighbor_temps = temperatures[neighbor_inds] mean_neighbor_temp = np.mean(neighbor_temps) std_neighbor_temp = np.std(neighbor_temps, ddof=1) if std_neighbor_temp == 0: std_neighbor_temp = 1e-6 # Avoid division by zero z_score = (temperatures[i] - mean_neighbor_temp) / std_neighbor_temp if abs(z_score) > z_threshold: is_outlier.append(True) else: is_outlier.append(False) else: is_outlier.append(False) data_gdf['Is_Outlier'] = is_outlier return data_gdf def get_elevation_at_points(geometry, dem_data=None, dem_transform=None): from rasterio.transform import rowcol if dem_data is not None and dem_transform is not None: elevations = [] for point in geometry: x, y = point.x, point.y try: row, col = rowcol(dem_transform, x, y) row = int(row) col = int(col) elevation = dem_data[row, col] elevations.append(elevation) except (IndexError, ValueError): elevations.append(np.nan) return elevations else: raise ValueError("Provide dem_data and dem_transform.") def get_interpolated_temperature_at_point(lat, lon, temperature_grid, transform): """ Given a latitude and longitude, return the interpolated temperature value at that point. Parameters: - lat, lon: Latitude and longitude of the point. - temperature_grid: The interpolated temperature grid (2D numpy array). - transform: Affine transform for the grid. Returns: - Interpolated temperature value at the point, or None if point is outside the grid. """ from rasterio.transform import rowcol try: row, col = rowcol(transform, lon, lat) # Note that rasterio uses (x, y), so (lon, lat) row = int(row) col = int(col) value = temperature_grid[row, col] if np.ma.is_masked(value) or np.isnan(value): # Value is masked or NaN, meaning outside the interpolated area return None else: return value.item() except (IndexError, ValueError): return None # Point is outside the grid def get_raster_value_at_points(geometry, raster_data=None, raster_transform=None): from rasterio.transform import rowcol if raster_data is not None and raster_transform is not None: values = [] for point in geometry: x, y = point.x, point.y try: row, col = rowcol(raster_transform, x, y) row = int(row) col = int(col) if (0 <= row < raster_data.shape[0]) and (0 <= col < raster_data.shape[1]): value = raster_data[row, col] # Check for NaN in raster data if np.isnan(value): values.append(np.nan) else: values.append(value) else: values.append(np.nan) except (IndexError, ValueError): values.append(np.nan) return values else: raise ValueError("Provide raster_data and raster_transform.") def get_water_body_at_points(geometry, water_data=None, water_transform=None): from rasterio.transform import rowcol if water_data is not None and water_transform is not None: water_bodies = [] for point in geometry: x, y = point.x, point.y try: row, col = rowcol(water_transform, x, y) row = int(row) col = int(col) if (0 <= row < water_data.shape[0]) and (0 <= col < water_data.shape[1]): value = water_data[row, col] # Adjust the condition based on your raster's water encoding # For binary raster where 1 = water, 0 = land water_bodies.append(1 if value else 0) else: water_bodies.append(0) except (IndexError, ValueError): water_bodies.append(0) return water_bodies else: raise ValueError("Provide water_data and water_transform.") def read_and_clip_dem(masovian_boundary): dem_path = 'input/masovian_dem.tif' with rasterio.open(dem_path) as src: print("Assuming DEM is in EPSG:4326; skipping reprojection.") dem_data = src.read(1, masked=True) dem_transform = src.transform return dem_data, dem_transform def read_and_clip_forest(masovian_boundary): forest_path = 'input/forests.tif' with rasterio.open(forest_path) as src: print("Assuming forest cover raster is in EPSG:4326; skipping reprojection.") forest_data = src.read(1, masked=True) forest_data = np.where(forest_data == src.nodata, 0, forest_data) forest_transform = src.transform return forest_data, forest_transform def read_and_clip_water_bodies(masovian_boundary): water_path = 'input/water_bodies.tif' with rasterio.open(water_path) as src: print("Reading and clipping water bodies raster to Masovian boundaries.") # Ensure the Masovian boundary is in the same CRS as the water raster if masovian_boundary.crs != src.crs: masovian_boundary = masovian_boundary.to_crs(src.crs) print("Reprojected Masovian boundary to match water body raster CRS.") else: print("Masovian boundary CRS matches water body raster CRS.") # Perform the clipping clipped_water, water_transform = rasterio.mask.mask( src, masovian_boundary.geometry, crop=True, filled=True, nodata=0 ) # Read the first band (assuming single-band raster) water_data = clipped_water[0] # Handle NoData values by setting them to 0 (assuming 0 represents no water) water_data = np.where(water_data == src.nodata, 0, water_data) return water_data, water_transform def read_and_clip_ecostress(masovian_boundary): ecostress_path = 'input/raw_ecostress.tif' with rasterio.open(ecostress_path) as src: print("Reading ECOSTRESS LST data (assumed to be in degrees Celsius).") # Read data ecostress_data = src.read(1, masked=True) ecostress_data = np.where(ecostress_data == src.nodata, np.nan, ecostress_data) ecostress_transform = src.transform return ecostress_data, ecostress_transform # Main execution flow if __name__ == "__main__": # Fetch Traxelektronik data trax_stations, trax_temperatures = fetch_traxelektronik_data() # Geocode Traxelektronik stations geolocator = Nominatim(user_agent="geoapi") trax_coords = {} progress_bar = tqdm(total=len(trax_stations), desc="Geocoding Traxelektronik Stations", unit="station") for station in trax_stations: cleaned_station = clean_station_name(station) coords, was_cleaned = get_coordinates(cleaned_station, geolocator) if coords: trax_coords[station] = coords progress_bar.update(1) progress_bar.close() # Fetch Netatmo data netatmo_stations, netatmo_temperatures, netatmo_coords = fetch_netatmo_data() # Fetch IMGW data imgw_stations, imgw_temperatures, imgw_station_ids = fetch_imgw_data() # Geocode IMGW stations imgw_coords = {} progress_bar = tqdm(total=len(imgw_stations), desc="Geocoding IMGW Stations", unit="station") for station in imgw_stations: coords, was_cleaned = get_coordinates(station, geolocator) if coords: imgw_coords[station] = coords progress_bar.update(1) progress_bar.close() # After plotting, get the interpolated temperature at specific coordinates # Coordinates to test test_lat = 51.40118 test_lon = 21.14805 smoothed_temperature, transform = plot_stations_with_boundary_and_interpolation( trax_stations, trax_temperatures, trax_coords, netatmo_stations, netatmo_temperatures, netatmo_coords, imgw_stations, imgw_temperatures, imgw_coords ) interpolated_temp = get_interpolated_temperature_at_point( test_lat, test_lon, smoothed_temperature, transform ) if interpolated_temp is not None: print(f"Interpolated temperature value at {test_lat}, {test_lon} is {interpolated_temp:.2f}°C") else: print(f"Coordinates {test_lat}, {test_lon} are outside the interpolated grid or have no data.")Explanation of Changes:
-
Error Fix: In the second plot, we created a new
OffsetImageinstance calledimagebox2to avoid reusing the same artist across figures. This prevents theRuntimeError. -
Spatial Outlier Detection: We introduced the
detect_local_outliersfunction, which uses spatial analysis to detect stations with anomalous temperature readings compared to their K nearest neighbors. -
Distance-Based Masking: We computed distances from each grid point to the nearest station and masked out areas where this distance exceeds a threshold (e.g., 25 km). This prevents interpolation in regions lacking data, reducing unnatural extrapolations.
These changes improve the robustness and accuracy of your interpolation process, especially in areas with sparse data or potential outlier stations.
Feel free to adjust parameters like
k,z_threshold, anddistance_thresholdbased on your specific data and requirements. -