Skip to content
The Chat Archive
o1-preview-2024-09-12· OpenAI WildChat-4.8M (AI2)

import geopandas as gpd import matplotlib.pyplot as plt from geopy.geocoders import…

  1. User

    import geopandas as gpd import matplotlib.pyplot as plt from geopy.geocoders import Nominatim import requests from bs4 import BeautifulSoup import re from tqdm import tqdm from shapely.geometry import Point import json import pandas as pd import numpy as np import rasterio import warnings from scipy.spatial import cKDTree from rasterio.transform import from_origin from rasterio.features import rasterize import os import rasterio.mask import matplotlib.colors as mcolors import matplotlib.patheffects as patheffects from pykrige.uk import UniversalKriging from sklearn.linear_model import HuberRegressor from scipy.stats import zscore from sklearn.ensemble import RandomForestRegressor from sklearn.preprocessing import StandardScaler from scipy.ndimage import gaussian_filter from datetime import datetime import pytz import math

    os.environ['PROJ_LIB'] = r'C:\Users\Lenovo\Documents\conda\Library\share\proj'

    warnings.filterwarnings("ignore")

    Keywords to remove

    keywords_to_remove = [ r"(centrum)", "rondo", "most", "Obwodnica", "obwodnica I", "obwodnica II", "obwodnica", "- Zawiszyn", "OUD Węzeł", "PPO", "Węzeł", "N (wąwóz)", "S (most)", "stacja", "z pomiarów", "min", "max", "średnia" ]

    def clean_station_name(station_name): pattern = re.compile("|".join(keywords_to_remove), re.IGNORECASE) cleaned_name = pattern.sub("", station_name).strip() return cleaned_name

    def get_masovian_boundary(): url = "https://raw.githubusercontent.com/ppatrzyk/polska-geojson/master/wojewodztwa/wojewodztwa-medium.geojson" # Fetch the content using requests response = requests.get(url) if response.status_code == 200: data = response.json() # Set the CRS to EPSG:4326 when creating the GeoDataFrame gdf = gpd.GeoDataFrame.from_features(data["features"], crs="EPSG:4326") masovian = gdf[gdf['nazwa'] == 'mazowieckie'] return masovian else: print(f"Error fetching GeoJSON data: {response.status_code}") return None

    Global variable for Masovian boundary

    masovian_boundary = get_masovian_boundary()

    def is_in_masovian(lat, lon): point = gpd.GeoSeries([Point(lon, lat)], crs="EPSG:4326") masovian_geom = masovian_boundary.to_crs(epsg=2180).reset_index(drop=True) point_projected = point.to_crs(epsg=2180) return masovian_geom.contains(point_projected.buffer(100)).values[0]

    def fetch_traxelektronik_data(): url = 'https://www.traxelektronik.pl/pogoda/zbiorcza.php?RejID=10' response = requests.get(url) soup = BeautifulSoup(response.text, 'html.parser')

    stations = []
    temperatures = []
    
    table_rows = soup.find_all('tr')[2:]
    
    for row in table_rows:
        columns = row.find_all('td')
        if len(columns) > 1:
            station = columns[0].get_text(strip=True)
            temperature = columns[1].get_text(strip=True)
            if temperature != "-":
                cleaned_station = clean_station_name(station)
                stations.append(cleaned_station)
                temperatures.append(temperature)
    
    print(f"Fetched {len(stations)} stations with valid temperature data from Traxelektronik.")
    return stations, temperatures
    

    def fetch_netatmo_data(): url = "https://api.netatmo.com/api/getpublicdata" params = { "lat_ne": 53, "lon_ne": 22, "lat_sw": 51, "lon_sw": 19, "required_data": "temperature", "filter": "true", "access_token": "66e1e80975134797710422cd|74965a60db5d205630cee60056012cf6" }

    response = requests.get(url, params=params)
    if response.status_code == 200:
        print("Successfully fetched data from Netatmo")
    else:
        print(f"Error fetching data: {response.status_code}")
    data = json.loads(response.text)
    
    stations = []
    temperatures = []
    coordinates = []
    
    for station in data.get('body', []):
        place = station.get('place', {})
        location = place.get('location')
        if location:
            lon, lat = location
            if is_in_masovian(lat, lon):
                for module, module_data in station.get('measures', {}).items():
                    if 'temperature' in module_data.get('type', []):
                        res = module_data.get('res', {})
                        if res:
                            latest_timestamp = max(res.keys())
                            temperature = res[latest_timestamp][0]
                            city = place.get('city', 'Unknown')
                            stations.append(city)
                            temperatures.append(temperature)
                            coordinates.append((lat, lon))
                            break
    
    print(f"Fetched {len(stations)} stations with valid temperature data from Netatmo in Masovian Voivodeship.")
    return stations, temperatures, coordinates
    

    def fetch_imgw_data(): url = "https://rafalraczynski.com.pl/imgw/dane-imgw/getJSON.php?type=table&province=14&sort=temp&order=asc" response = requests.get(url) if response.status_code == 200: data = response.json() else: print(f"Error fetching IMGW data: {response.status_code}") return [], [], []

    target_stations = [
        "ANDRZEJEWO", "BORKOWO", "CZARNOWO", "KAZANÓW",
        "RUSZKOWICE", "RYBIENKO", "RZĄŚNIK WŁOŚCIAŃSKI", "WIELGOLAS"
    ]
    
    stations = []
    temperatures = []
    station_ids = []
    
    for item in data:
        statName = item.get('statName')
        temp = item.get('temp')
        if statName and temp and statName.upper() in target_stations:
            stations.append(statName)
            temperatures.append(temp)
            station_ids.append(item.get('statId', 'Unknown'))
    
    print(f"Fetched {len(stations)} stations from IMGW data.")
    return stations, temperatures, station_ids
    

    def get_coordinates(station_name, geolocator): try: location = geolocator.geocode(f"{station_name}, Masovian Voivodeship, Poland") if location: if is_in_masovian(location.latitude, location.longitude): print(f"Geocoded '{station_name}' to {location.latitude}, {location.longitude} (in Masovian).") return (location.latitude, location.longitude), False else: print(f"'{station_name}' is not in Masovian Voivodeship.") return None, False print(f"Failed to geocode '{station_name}'.") return None, False except Exception as e: print(f"Error geocoding {station_name}: {e}") return None, False

    def clean_temperature(temp): cleaned_temp = re.sub(r'[↓↑\s]', '', temp) try: return float(cleaned_temp) except ValueError: return None

    def remove_nearby_duplicates(trax_coords, trax_temperatures, netatmo_coords, netatmo_temperatures, distance_threshold=0.01): trax_points = np.array([(lon, lat) for station, (lat, lon) in trax_coords.items()]) netatmo_points = np.array([(lon, lat) for lat, lon in netatmo_coords]) trax_tree = cKDTree(trax_points)

    netatmo_filtered_coords = []
    netatmo_filtered_temperatures = []
    
    for netatmo_point, netatmo_temp in zip(netatmo_points, netatmo_temperatures):
        distance, index = trax_tree.query(netatmo_point)
        if distance > distance_threshold:
            netatmo_filtered_coords.append(tuple(netatmo_point))
            netatmo_filtered_temperatures.append(netatmo_temp)
    
    return netatmo_filtered_coords, netatmo_filtered_temperatures
    

    def hillshade(array, azimuth=315, angle_altitude=45): # Convert angles to radians azimuth_rad = np.radians(azimuth) altitude_rad = np.radians(angle_altitude)

    # Calculate gradients in x and y directions
    x, y = np.gradient(array)
    
    # Calculate the slope and aspect
    slope = np.pi/2.0 - np.arctan(np.sqrt(x*x + y*y))
    aspect = np.arctan2(-x, y)
    
    # Calculate shaded relief
    shaded = np.sin(altitude_rad) * np.sin(slope) + np.cos(altitude_rad) * np.cos(slope) * np.cos(azimuth_rad - aspect)
    
    # Normalize the shaded relief
    shaded = (shaded + 1) / 2
    shaded = (shaded * 255).astype(np.uint8)
    
    return shaded
    

    def gaussian_filter_nan(image, sigma): nan_mask = np.isnan(image) image_filled = np.where(nan_mask, 0, image) weights = np.where(nan_mask, 0, 1) image_filtered = gaussian_filter(image_filled, sigma=sigma, mode='nearest') weights_filtered = gaussian_filter(weights, sigma=sigma, mode='nearest') with np.errstate(divide='ignore', invalid='ignore'): image_final = image_filtered / weights_filtered image_final[nan_mask] = np.nan return image_final

    def plot_stations_with_boundary_and_interpolation( trax_stations, trax_temperatures, trax_coords, netatmo_stations, netatmo_temperatures, netatmo_coords, imgw_stations, imgw_temperatures, imgw_coords): # Process Traxelektronik data trax_filtered_stations = [] trax_filtered_temperatures = [] trax_geometries = []

    # Filter and clean Traxelektronik data
    for station, temperature in zip(trax_stations, trax_temperatures):
        cleaned_station = clean_station_name(station)
        cleaned_temperature = clean_temperature(temperature)
        if station in trax_coords and cleaned_temperature is not None:
            lat, lon = trax_coords[station]
            trax_filtered_stations.append(cleaned_station)
            trax_filtered_temperatures.append(cleaned_temperature)
            trax_geometries.append(Point(lon, lat))
    
    # Create GeoDataFrame for Traxelektronik data
    trax_gdf = gpd.GeoDataFrame({
        'Station': trax_filtered_stations,
        'Temperature': trax_filtered_temperatures,
        'Source': 'Traxelektronik'
    }, geometry=trax_geometries, crs="EPSG:4326")
    
    # Handle Netatmo data
    netatmo_filtered_coords = []
    netatmo_filtered_temperatures = []
    netatmo_filtered_stations = []
    
    for station, temperature, coord in zip(netatmo_stations, netatmo_temperatures, netatmo_coords):
        if isinstance(temperature, (int, float)) and coord is not None:
            netatmo_filtered_coords.append(coord)
            netatmo_filtered_temperatures.append(temperature)
            netatmo_filtered_stations.append(station)
    
    if len(netatmo_filtered_coords) > 0:
        netatmo_geometries = [Point(lon, lat) for lat, lon in netatmo_filtered_coords]
        netatmo_gdf = gpd.GeoDataFrame({
            'Station': netatmo_filtered_stations,
            'Temperature': netatmo_filtered_temperatures,
            'Source': 'Netatmo'
        }, geometry=netatmo_geometries, crs="EPSG:4326")
    else:
        netatmo_gdf = gpd.GeoDataFrame(columns=['Station', 'Temperature', 'Source', 'geometry'], crs="EPSG:4326")
    
    # Handle IMGW data
    imgw_filtered_stations = []
    imgw_filtered_temperatures = []
    imgw_geometries = []
    
    # Clean and process IMGW data
    for station, temperature in zip(imgw_stations, imgw_temperatures):
        cleaned_temperature = clean_temperature(temperature)
        if station in imgw_coords and cleaned_temperature is not None:
            lat, lon = imgw_coords[station]
            imgw_filtered_stations.append(station)
            imgw_filtered_temperatures.append(cleaned_temperature)
            imgw_geometries.append(Point(lon, lat))
    
    # Create GeoDataFrame for IMGW data
    imgw_gdf = gpd.GeoDataFrame({
        'Station': imgw_filtered_stations,
        'Temperature': imgw_filtered_temperatures,
        'Source': 'IMGW'
    }, geometry=imgw_geometries, crs="EPSG:4326")
    
    # Read and clip DEM
    dem_data, dem_transform = read_and_clip_dem(masovian_boundary)
    
    # Read and clip Forest Cover
    forest_data, forest_transform = read_and_clip_forest(masovian_boundary)
    
    # Read and clip Water Bodies
    water_data, water_transform = read_and_clip_water_bodies(masovian_boundary)
    
    # Read ECOSTRESS data
    ecostress_data, ecostress_transform = read_and_clip_ecostress(masovian_boundary)
    
    # Ensure geometries are in EPSG:4326
    trax_gdf_dem_crs = trax_gdf.to_crs("EPSG:4326")
    netatmo_gdf_dem_crs = netatmo_gdf.to_crs("EPSG:4326")
    imgw_gdf_dem_crs = imgw_gdf.to_crs("EPSG:4326")
    
    # Get elevation at station points using dem_data and dem_transform
    trax_elevations = get_elevation_at_points(
        trax_gdf_dem_crs.geometry,
        dem_data=dem_data,
        dem_transform=dem_transform
    )
    netatmo_elevations = get_elevation_at_points(
        netatmo_gdf_dem_crs.geometry,
        dem_data=dem_data,
        dem_transform=dem_transform
    )
    imgw_elevations = get_elevation_at_points(
        imgw_gdf_dem_crs.geometry,
        dem_data=dem_data,
        dem_transform=dem_transform
    )
    
    # Add elevation to the GeoDataFrames
    trax_gdf['Elevation'] = trax_elevations
    netatmo_gdf['Elevation'] = netatmo_elevations
    imgw_gdf['Elevation'] = imgw_elevations
    
    # Get tree cover density at station points using forest_data and forest_transform
    trax_forest = get_raster_value_at_points(
        trax_gdf_dem_crs.geometry,
        raster_data=forest_data,
        raster_transform=forest_transform
    )
    netatmo_forest = get_raster_value_at_points(
        netatmo_gdf_dem_crs.geometry,
        raster_data=forest_data,
        raster_transform=forest_transform
    )
    imgw_forest = get_raster_value_at_points(
        imgw_gdf_dem_crs.geometry,
        raster_data=forest_data,
        raster_transform=forest_transform
    )
    
    # Add tree cover density to the GeoDataFrames
    trax_gdf['Tree_Cover'] = trax_forest
    netatmo_gdf['Tree_Cover'] = netatmo_forest
    imgw_gdf['Tree_Cover'] = imgw_forest
    
    # Get water body data at station points
    trax_water = get_water_body_at_points(
        trax_gdf_dem_crs.geometry,
        water_data=water_data,
        water_transform=water_transform
    )
    netatmo_water = get_water_body_at_points(
        netatmo_gdf_dem_crs.geometry,
        water_data=water_data,
        water_transform=water_transform
    )
    imgw_water = get_water_body_at_points(
        imgw_gdf_dem_crs.geometry,
        water_data=water_data,
        water_transform=water_transform
    )
    
    # Add water body data to the GeoDataFrames
    trax_gdf['Water_Body'] = trax_water
    netatmo_gdf['Water_Body'] = netatmo_water
    imgw_gdf['Water_Body'] = imgw_water
    
    # Add 'Priority' column to each GeoDataFrame
    trax_gdf['Priority'] = 2
    netatmo_gdf['Priority'] = 1
    imgw_gdf['Priority'] = 3  # Higher priority
    
    # Combine GeoDataFrames
    stations_gdf = gpd.GeoDataFrame(pd.concat([trax_gdf, netatmo_gdf, imgw_gdf], ignore_index=True))
    stations_gdf.crs = "EPSG:4326"
    stations_gdf = stations_gdf.dropna(subset=['Temperature', 'Elevation', 'Tree_Cover'])
    
    # Extract coordinates
    stations_gdf['X'] = stations_gdf.geometry.x
    stations_gdf['Y'] = stations_gdf.geometry.y
    
    # Prepare data for regression
    X = stations_gdf[['X', 'Y', 'Elevation', 'Tree_Cover', 'Water_Body']]
    y = stations_gdf['Temperature']
    
    # Normalize covariates
    scaler = StandardScaler()
    X_scaled = scaler.fit_transform(X)
    
    # Fit Random Forest Regression model
    print("Training Random Forest Regression model...")
    rf_model = RandomForestRegressor(n_estimators=100, random_state=42)
    rf_model.fit(X_scaled, y)
    
    # Predict temperatures and compute residuals
    stations_gdf['Predicted_Temperature'] = rf_model.predict(X_scaled)
    stations_gdf['Residual'] = stations_gdf['Temperature'] - stations_gdf['Predicted_Temperature']
    
    # Outlier detection using Z-score
    stations_gdf['Residual_Z'] = zscore(stations_gdf['Residual'])
    z_threshold = 2  # Set your threshold
    stations_gdf_inliers = stations_gdf[np.abs(stations_gdf['Residual_Z']) <= z_threshold].copy()
    
    # Remove duplicates by prioritizing IMGW data
    data = stations_gdf_inliers.copy()
    
    # Sort by priority (higher priority first)
    data = data.sort_values('Priority', ascending=False)
    
    # Group by coordinates and keep the first record in each group (highest priority)
    data = data.groupby(['X', 'Y'], as_index=False).first()
    
    unique_points = data[['X', 'Y']].values
    unique_residuals = data['Residual'].values
    unique_elevation = data['Elevation'].values
    unique_tree_cover = data['Tree_Cover'].values
    unique_water_body = data['Water_Body'].values
    
    # Create grid in EPSG:4326 using rasterio's transform functions
    # Adjust grid_spacing as per your requirements
    grid_spacing = 0.0025  # Approximately ~5 km
    
    x_min, y_min, x_max, y_max = masovian_boundary.total_bounds
    
    # Compute the number of cols and rows with the new grid_spacing
    ncols = int(np.ceil((x_max - x_min) / grid_spacing))
    nrows = int(np.ceil((y_max - y_min) / grid_spacing))
    
    # Create the affine transform for the grid
    transform = from_origin(x_min, y_max, grid_spacing, grid_spacing)
    
    # Create arrays of row and column indices
    cols = np.arange(ncols)
    rows = np.arange(nrows)
    
    # Create a meshgrid of indices
    cols_indices, rows_indices = np.meshgrid(cols, rows)
    
    # Flatten the indices
    cols_flat = cols_indices.ravel()
    rows_flat = rows_indices.ravel()
    
    # Get the x and y coordinates for each grid cell center
    xs_flat, ys_flat = rasterio.transform.xy(transform, rows_flat, cols_flat)
    
    # Create grid dataframe
    grid_df = pd.DataFrame({
        'X': xs_flat,
        'Y': ys_flat,
        'row_idx': rows_flat,
        'col_idx': cols_flat
    })
    
    # Retrieve elevations from DEM
    grid_elevations = get_elevation_at_points(
        [Point(x, y) for x, y in zip(grid_df['X'], grid_df['Y'])],
        dem_data=dem_data,
        dem_transform=dem_transform
    )
    grid_df['Elevation'] = grid_elevations
    
    # Retrieve tree cover density from forest cover raster
    grid_forest = get_raster_value_at_points(
        [Point(x, y) for x, y in zip(grid_df['X'], grid_df['Y'])],
        raster_data=forest_data,
        raster_transform=forest_transform
    )
    grid_df['Tree_Cover'] = grid_forest
    
    # Retrieve water body data at grid points
    grid_water = get_water_body_at_points(
        [Point(x, y) for x, y in zip(grid_df['X'], grid_df['Y'])],
        water_data=water_data,
        water_transform=water_transform
    )
    grid_df['Water_Body'] = grid_water
    
    # Remove grid points with NaN values (i.e., outside data coverage)
    grid_df = grid_df.dropna(subset=['Elevation', 'Tree_Cover'])
    
    # Prepare data for regression prediction
    X_grid = grid_df[['X', 'Y', 'Elevation', 'Tree_Cover', 'Water_Body']]
    X_grid_scaled = scaler.transform(X_grid)
    grid_df['Predicted_Temperature'] = rf_model.predict(X_grid_scaled)
    
    # Set up variogram model with adjusted parameters
    variogram_model = 'spherical'
    # Adjust these parameters based on your data; you'll need to experiment with them
    variogram_parameters = {'nugget': 0.2, 'sill': 1.0, 'range': 0.05}
    
    # Perform Universal Kriging
    print("Performing Universal Kriging interpolation for residuals with adjusted variogram parameters...")
    uk = UniversalKriging(
        x=data['X'].values,
        y=data['Y'].values,
        z=data['Residual'].values,
        variogram_model=variogram_model,
        variogram_parameters=variogram_parameters,
        nlags=20,
        verbose=True
    )
    
    z, ss = uk.execute(
        'points',
        grid_df['X'].values,
        grid_df['Y'].values
    )
    
    grid_df['Residual'] = z
    grid_df['Kriging_Variance'] = ss
    
    # Final temperature estimation
    grid_df['Temperature'] = grid_df['Predicted_Temperature'] + grid_df['Residual']
    
    # Save a copy of the temperature before ECOSTRESS adjustments
    grid_df['Temperature_No_ECOSTRESS'] = grid_df['Temperature'].copy()
    
    # ECOSTRESS Adjustments Begin Here
    
    # Retrieve ECOSTRESS LST values at grid points (same as before)
    grid_ecostress = get_raster_value_at_points(
        [Point(x, y) for x, y in zip(grid_df['X'], grid_df['Y'])],
        raster_data=ecostress_data,
        raster_transform=ecostress_transform
    )
    grid_df['ECOSTRESS_LST'] = grid_ecostress
    
    # Handle NaNs in ECOSTRESS_LST by filling with the mean value
    grid_df['ECOSTRESS_LST'].fillna(grid_df['ECOSTRESS_LST'].mean(), inplace=True)
    
    # Compute ECOSTRESS anomalies (deviation from mean)
    ecostress_mean = grid_df['ECOSTRESS_LST'].mean()
    grid_df['ECOSTRESS_Anomaly'] = grid_df['ECOSTRESS_LST'] - ecostress_mean
    
    # Normalize anomalies to range [-1, 1]
    ecostress_anomaly_min = grid_df['ECOSTRESS_Anomaly'].min()
    ecostress_anomaly_max = grid_df['ECOSTRESS_Anomaly'].max()
    grid_df['ECOSTRESS_Anomaly_Normalized'] = (
        (grid_df['ECOSTRESS_Anomaly'] - ecostress_anomaly_min) / (ecostress_anomaly_max - ecostress_anomaly_min) * 2 - 1
    )
    
    # Increase the impact of ECOSTRESS LST data
    amplification_factor = 3.0  # Adjust as needed
    
    # Determine the current time in CET
    cet = pytz.timezone('CET')
    now_cet = datetime.now(cet)
    hour = now_cet.hour + now_cet.minute / 60.0
    
    # Gaussian weighting centered at 2 PM with a spread over daylight hours
    hour_peak = 14  # Peak hour (2 PM)
    sigma_hour = 5  # Standard deviation in hours
    
    # Compute day_weight using the Gaussian function
    day_weight = math.exp(-((hour - hour_peak) ** 2) / (2 * sigma_hour ** 2))
    
    # Apply anomalies with the new day_weight
    adjustment = grid_df['ECOSTRESS_Anomaly_Normalized'] * amplification_factor * day_weight
    
    # Prevent ECOSTRESS adjustments on water bodies
    adjustment[grid_df['Water_Body'] == 1] = 0
    
    # Limit maximum adjustment to avoid over-adjusting
    max_adjustment = 5.0  # degrees Celsius
    adjustment = adjustment.clip(-max_adjustment, max_adjustment)
    
    # Apply adjustments to the interpolated temperatures
    grid_df['Temperature'] += adjustment
    
    # Apply cooling effect on water bodies
    cooling_effect = -2.0  # degrees Celsius
    grid_df.loc[grid_df['Water_Body'] == 1, 'Temperature'] += cooling_effect
    
    # Remove Gaussian smoothing from the interpolated temperature grid
    
    # First, create 2D arrays of temperatures without smoothing
    grid_temperature_no_ecostress = np.full((nrows, ncols), np.nan)
    grid_temperature_no_ecostress[grid_df['row_idx'], grid_df['col_idx']] = grid_df['Temperature_No_ECOSTRESS'].values
    
    grid_temperature_with_ecostress = np.full((nrows, ncols), np.nan)
    grid_temperature_with_ecostress[grid_df['row_idx'], grid_df['col_idx']] = grid_df['Temperature'].values
    
    # Mask the grid_temperature with the Masovian boundary
    masovian_shape = [masovian_boundary.geometry.unary_union]
    mask = rasterize(
        masovian_shape,
        out_shape=(nrows, ncols),
        transform=transform,
        fill=0,
        all_touched=True,
        default_value=1,
        dtype='uint8'
    )
    
    # Apply the mask
    grid_temperature_no_ecostress = np.ma.array(grid_temperature_no_ecostress, mask=mask == 0)
    grid_temperature_with_ecostress = np.ma.array(grid_temperature_with_ecostress, mask=mask == 0)
    
    # Use the unsmoothed temperature grids
    temperature_no_ecostress = grid_temperature_no_ecostress
    temperature_with_ecostress = grid_temperature_with_ecostress
    
    # Also create grid_variance array (we'll skip smoothing variance for simplicity)
    grid_variance = np.full((nrows, ncols), np.nan)
    grid_variance[grid_df['row_idx'], grid_df['col_idx']] = grid_df['Kriging_Variance']
    grid_variance = np.ma.array(grid_variance, mask=mask == 0)
    
    # Compute Standard Deviation from Variance
    grid_std_dev = np.sqrt(grid_variance)
    grid_std_dev = np.ma.array(grid_std_dev, mask=mask == 0)
    
    # Read the custom color scale from the CSV file
    color_scale_df = pd.read_csv('input/color_scale.csv')
    color_scale_df = color_scale_df.sort_values('value')
    values = color_scale_df['value'].values
    hex_colors = color_scale_df['color'].values
    
    # Ensure colors start with '#'
    hex_colors = [color if color.startswith('#') else f"#{color}" for color in hex_colors]
    
    # Create a custom colormap
    # Normalize the values to 0..1
    vmin = values.min()
    vmax = values.max()
    normalized_values = (values - vmin) / (vmax - vmin)
    
    # Create a list of tuples (position, color)
    color_tuples = list(zip(normalized_values, hex_colors))
    
    # Create a LinearSegmentedColormap
    cmap = mcolors.LinearSegmentedColormap.from_list('custom_cmap', color_tuples)
    
    # Set the normalization
    norm = mcolors.Normalize(vmin=vmin, vmax=vmax)
    
    with rasterio.open('input/masovian_dem.tif') as dem_src:
        # Reproject Masovian boundary to DEM CRS if necessary
        if masovian_boundary.crs != dem_src.crs:
            masovian_boundary_dem_crs = masovian_boundary.to_crs(dem_src.crs)
        else:
            masovian_boundary_dem_crs = masovian_boundary
    
        # Clip the DEM to the Masovian boundary
        dem_plot_data, dem_plot_transform = rasterio.mask.mask(
            dem_src, masovian_boundary_dem_crs.geometry, crop=True, filled=True, nodata=np.nan
        )
    
        # Read the first band (assuming single-band DEM)
        dem_plot_data = dem_plot_data[0]  # Since mask returns data as (bands, rows, cols)
    
        # Handle NaNs in the DEM data
        dem_plot_data = np.nan_to_num(dem_plot_data, nan=np.nanmin(dem_plot_data))
    
        # Calculate the extent of the clipped DEM for plotting
        dem_left, dem_top = dem_plot_transform * (0, 0)
        dem_right, dem_bottom = dem_plot_transform * (dem_plot_data.shape[1], dem_plot_data.shape[0])
        dem_extent = [dem_left, dem_right, dem_bottom, dem_top]
    
    # Compute extent for plotting (same as before)
    def get_extent(transform, array):
        x_min = transform.c
        y_max = transform.f
        x_max = x_min + transform.a * array.shape[1]
        y_min = y_max + transform.e * array.shape[0]
        return [x_min, x_max, y_min, y_max]
    
    hs = hillshade(dem_plot_data, azimuth=315, angle_altitude=45)
    
    extent = get_extent(transform, temperature_with_ecostress)
    dem_extent = get_extent(dem_plot_transform, dem_plot_data)
    
    # Plotting Temperature Map without ECOSTRESS Adjustments
    fig1, ax1 = plt.subplots(figsize=(15, 15))
    
    # Set modern and minimalistic font properties with increased size
    plt.rcParams.update({
        'font.size': 10,
        'font.family': 'sans-serif',
        'font.sans-serif': ['Arial', 'Helvetica', 'DejaVu Sans']
    })
    
    # Plot the interpolated temperature grid without ECOSTRESS adjustments
    img1 = ax1.imshow(temperature_no_ecostress, extent=extent, origin='upper',
                      cmap=cmap, norm=norm, alpha=1)
    
    # Plot hillshade on top
    ax1.imshow(hs, extent=dem_extent, cmap='gray', origin='upper', alpha=0.3)
    
    # Overlay water bodies with blue color
    water_mask = np.where(water_data > 0, 1, 0)  # Adjust condition based on water_data values
    ax1.imshow(water_mask, extent=extent, origin='upper', cmap='Blues', alpha=0.5)  # Increased alpha to 0.5
    
    # Plot Masovian Voivodeship boundary
    masovian_boundary.boundary.plot(ax=ax1, color='black', linewidth=2)
    
    # Plot stations as smaller rectangles
    stations_gdf.plot(ax=ax1, color='black', marker='s', markersize=5, label='Stations')
    
    # Add contours for every 1°C value
    min_temp = np.floor(temperature_no_ecostress.min())
    max_temp = np.ceil(temperature_no_ecostress.max())
    contour_levels = np.arange(min_temp, max_temp + 1, 1)
    
    # Add contours
    contours1 = ax1.contour(
        temperature_no_ecostress, levels=contour_levels, colors='black', linewidths=0.5,
        extent=extent, origin='upper'
    )
    contour_labels1 = ax1.clabel(
        contours1, inline=False, fontsize=8, fmt='%d', colors='white',
        inline_spacing=4, manual=False
    )
    for txt in contour_labels1:
        txt.set_path_effects([
            patheffects.Stroke(linewidth=3, foreground='black'),
            patheffects.Normal()
        ])
    
    # Add colorbar and labels
    cbar1 = fig1.colorbar(img1, ax=ax1, label="Temperature (°C)")
    ticks = np.arange(np.floor(vmin / 5) * 5, np.ceil(vmax / 5) * 5 + 1, 5)
    cbar1.set_ticks(ticks)
    cbar1.set_ticklabels(ticks.astype(int))
    
    ax1.set_title("Temperature Interpolation without ECOSTRESS Adjustments", fontsize=18)
    ax1.legend()
    ax1.set_xlabel("Longitude")
    ax1.set_ylabel("Latitude")
    
    plt.savefig('output/temperature_without_ecostress.png', dpi=300, bbox_inches='tight')
    plt.show()
    
    print(f"Temperature interpolation plot created successfully without smoothing.")
    
    # Plotting Temperature Map with ECOSTRESS Adjustments
    fig2, ax2 = plt.subplots(figsize=(15, 15))
    
    # Plot the interpolated temperature grid with ECOSTRESS adjustments
    img2 = ax2.imshow(temperature_with_ecostress, extent=extent, origin='upper',
                      cmap=cmap, norm=norm, alpha=1)
    
    # Plot hillshade on top
    ax2.imshow(hs, extent=dem_extent, cmap='gray', origin='upper', alpha=0.3)
    
    # Overlay water bodies
    ax2.imshow(water_mask, extent=extent, origin='upper', cmap='Blues', alpha=0.5)
    
    # Plot Masovian Voivodeship boundary
    masovian_boundary.boundary.plot(ax=ax2, color='black', linewidth=2)
    
    # Plot stations
    stations_gdf.plot(ax=ax2, color='black', marker='s', markersize=5, label='Stations')
    
    # Add contours
    contours2 = ax2.contour(
        temperature_with_ecostress, levels=contour_levels, colors='black', linewidths=0.5,
        extent=extent, origin='upper'
    )
    contour_labels2 = ax2.clabel(
        contours2, inline=False, fontsize=8, fmt='%d', colors='white',
        inline_spacing=4, manual=False
    )
    for txt in contour_labels2:
        txt.set_path_effects([
            patheffects.Stroke(linewidth=3, foreground='black'),
            patheffects.Normal()
        ])
    
    # Add colorbar and labels
    cbar2 = fig2.colorbar(img2, ax=ax2, label="Temperature (°C)")
    cbar2.set_ticks(ticks)
    cbar2.set_ticklabels(ticks.astype(int))
    
    ax2.set_title("Temperature Interpolation with ECOSTRESS Adjustments", fontsize=18)
    ax2.legend()
    ax2.set_xlabel("Longitude")
    ax2.set_ylabel("Latitude")
    
    plt.savefig('output/temperature_with_ecostress.png', dpi=300, bbox_inches='tight')
    plt.show()
    
    print(f"Temperature interpolation plot with ECOSTRESS adjustments created successfully without smoothing.")
    
    # Plotting the Difference Map
    # Compute the difference between the two temperature grids
    temperature_difference = temperature_with_ecostress - temperature_no_ecostress
    
    fig3, ax3 = plt.subplots(figsize=(15, 15))
    
    # Use a diverging colormap to show positive and negative differences
    diff_cmap = 'seismic'
    diff_norm = mcolors.TwoSlopeNorm(
        vmin=-np.max(np.abs(temperature_difference)),
        vcenter=0,
        vmax=np.max(np.abs(temperature_difference))
    )
    
    # Plot the temperature difference grid
    img3 = ax3.imshow(temperature_difference, extent=extent, origin='upper',
                      cmap=diff_cmap, norm=diff_norm, alpha=1)
    
    # Plot hillshade
    ax3.imshow(hs, extent=dem_extent, cmap='gray', origin='upper', alpha=0.3)
    
    # Overlay water bodies
    ax3.imshow(water_mask, extent=extent, origin='upper', cmap='Blues', alpha=0.5)
    
    # Plot Masovian Voivodeship boundary
    masovian_boundary.boundary.plot(ax=ax3, color='black', linewidth=2)
    
    # Plot stations
    stations_gdf.plot(ax=ax3, color='black', marker='s', markersize=5, label='Stations')
    
    # Add colorbar
    cbar3 = fig3.colorbar(img3, ax=ax3, label="Temperature Difference (°C)")
    
    ax3.set_title("Difference Due to ECOSTRESS Adjustments", fontsize=18)
    ax3.set_xlabel("Longitude")
    ax3.set_ylabel("Latitude")
    
    plt.savefig('output/temperature_difference.png', dpi=300, bbox_inches='tight')
    plt.show()
    
    print(f"Temperature difference plot created successfully without smoothing.")
    
    return temperature_with_ecostress, transform
    

    def get_elevation_at_points(geometry, dem_data=None, dem_transform=None): from rasterio.transform import rowcol

    if dem_data is not None and dem_transform is not None:
        elevations = []
        for point in geometry:
            x, y = point.x, point.y
            try:
                row, col = rowcol(dem_transform, x, y)
                row = int(row)
                col = int(col)
                elevation = dem_data[row, col]
                elevations.append(elevation)
            except (IndexError, ValueError):
                elevations.append(np.nan)
        return elevations
    else:
        raise ValueError("Provide dem_data and dem_transform.")
    

    def get_interpolated_temperature_at_point(lat, lon, temperature_grid, transform): """ Given a latitude and longitude, return the interpolated temperature value at that point.

    Parameters:
    - lat, lon: Latitude and longitude of the point.
    - temperature_grid: The interpolated temperature grid (2D numpy array).
    - transform: Affine transform for the grid.
    
    Returns:
    - Interpolated temperature value at the point, or None if point is outside the grid.
    """
    from rasterio.transform import rowcol
    
    try:
        row, col = rowcol(transform, lon, lat)  # Note that rasterio uses (x, y), so (lon, lat)
        row = int(row)
        col = int(col)
        value = temperature_grid[row, col]
        if np.ma.is_masked(value) or np.isnan(value):
            # Value is masked or NaN, meaning outside the interpolated area
            return None
        else:
            return value.item()
    except (IndexError, ValueError):
        return None  # Point is outside the grid
    

    def get_raster_value_at_points(geometry, raster_data=None, raster_transform=None): from rasterio.transform import rowcol

    if raster_data is not None and raster_transform is not None:
        values = []
        for point in geometry:
            x, y = point.x, point.y
            try:
                row, col = rowcol(raster_transform, x, y)
                row = int(row)
                col = int(col)
                value = raster_data[row, col]
                values.append(value)
            except (IndexError, ValueError):
                values.append(np.nan)
        return values
    else:
        raise ValueError("Provide raster_data and raster_transform.")
    

    def get_water_body_at_points(geometry, water_data=None, water_transform=None): from rasterio.transform import rowcol

    if water_data is not None and water_transform is not None:
        water_bodies = []
        for point in geometry:
            x, y = point.x, point.y
            try:
                row, col = rowcol(water_transform, x, y)
                row = int(row)
                col = int(col)
                if (0 <= row < water_data.shape[0]) and (0 <= col < water_data.shape[1]):
                    value = water_data[row, col]
                    # Adjust the condition based on your raster's water encoding
                    # For binary raster where 1 = water, 0 = land
                    water_bodies.append(1 if value else 0)
                else:
                    water_bodies.append(0)
            except (IndexError, ValueError):
                water_bodies.append(0)
        return water_bodies
    else:
        raise ValueError("Provide water_data and water_transform.")
    

    def read_and_clip_dem(masovian_boundary): dem_path = 'input/masovian_dem.tif' with rasterio.open(dem_path) as src: print("Assuming DEM is in EPSG:4326; skipping reprojection.") dem_data = src.read(1, masked=True) dem_transform = src.transform return dem_data, dem_transform

    def read_and_clip_forest(masovian_boundary): forest_path = 'input/forests.tif' with rasterio.open(forest_path) as src: print("Assuming forest cover raster is in EPSG:4326; skipping reprojection.") forest_data = src.read(1, masked=True) forest_data = np.where(forest_data == src.nodata, 0, forest_data) forest_transform = src.transform return forest_data, forest_transform

    def read_and_clip_water_bodies(masovian_boundary): water_path = 'input/water_bodies.tif' with rasterio.open(water_path) as src: print("Reading and clipping water bodies raster to Masovian boundaries.")

        # Ensure the Masovian boundary is in the same CRS as the water raster
        if masovian_boundary.crs != src.crs:
            masovian_boundary = masovian_boundary.to_crs(src.crs)
            print("Reprojected Masovian boundary to match water body raster CRS.")
        else:
            print("Masovian boundary CRS matches water body raster CRS.")
        
        # Perform the clipping
        clipped_water, water_transform = rasterio.mask.mask(
            src, masovian_boundary.geometry, crop=True, filled=True, nodata=0
        )
        
        # Read the first band (assuming single-band raster)
        water_data = clipped_water[0]
        
        # Handle NoData values by setting them to 0 (assuming 0 represents no water)
        water_data = np.where(water_data == src.nodata, 0, water_data)
        
    return water_data, water_transform
    

    Add the following function to read and clip the ECOSTRESS data

    def read_and_clip_ecostress(masovian_boundary): ecostress_path = 'input/raw_ecostress.tif' with rasterio.open(ecostress_path) as src: print("Reading ECOSTRESS LST data (assumed to be in degrees Celsius).") # Read data ecostress_data = src.read(1, masked=True) ecostress_data = np.where(ecostress_data == src.nodata, 0, ecostress_data) ecostress_transform = src.transform return ecostress_data, ecostress_transform

    Main execution flow

    if name == "main": # Fetch Traxelektronik data trax_stations, trax_temperatures = fetch_traxelektronik_data()

    # Geocode Traxelektronik stations
    geolocator = Nominatim(user_agent="geoapi")
    trax_coords = {}
    
    progress_bar = tqdm(total=len(trax_stations), desc="Geocoding Traxelektronik Stations", unit="station")
    for station in trax_stations:
        cleaned_station = clean_station_name(station)
        coords, was_cleaned = get_coordinates(cleaned_station, geolocator)
        if coords:
            trax_coords[station] = coords
        progress_bar.update(1)
    progress_bar.close()
    
    # Fetch Netatmo data
    netatmo_stations, netatmo_temperatures, netatmo_coords = fetch_netatmo_data()
    
    # Fetch IMGW data
    imgw_stations, imgw_temperatures, imgw_station_ids = fetch_imgw_data()
    
    # Geocode IMGW stations
    imgw_coords = {}
    progress_bar = tqdm(total=len(imgw_stations), desc="Geocoding IMGW Stations", unit="station")
    for station in imgw_stations:
        coords, was_cleaned = get_coordinates(station, geolocator)
        if coords:
            imgw_coords[station] = coords
        progress_bar.update(1)
    progress_bar.close()
    
    # After plotting, get the interpolated temperature at specific coordinates
    # Coordinates to test
    test_lat = 51.40118
    test_lon = 21.14805
    
    smoothed_temperature, transform = plot_stations_with_boundary_and_interpolation(
        trax_stations, trax_temperatures, trax_coords,
        netatmo_stations, netatmo_temperatures, netatmo_coords,
        imgw_stations, imgw_temperatures, imgw_coords
    )
    
    interpolated_temp = get_interpolated_temperature_at_point(
        test_lat, test_lon, smoothed_temperature, transform
    )
    
    if interpolated_temp is not None:
        print(f"Interpolated temperature value at {test_lat}, {test_lon} is {interpolated_temp:.2f}°C")
    else:
        print(f"Coordinates {test_lat}, {test_lon} are outside the interpolated grid or have no data.")
    

    Can you also make there so the deeper it's into night the colder will adjustments based on ecostress data get, but only in these the coldest areas (not in every areas below 0 degrees adjustments in the final adjustments plot)? After the deep night the adjustments temperature should start slowly, smoothly rising following further sunrise

  2. Assistant

    Certainly! Based on your request, you want to adjust the ECOSTRESS-based temperature adjustments so that:

    1. During the deep night, the coldest areas become even colder due to the adjustments.
    2. After the deep night, as the sun rises, the adjustments should smoothly decrease, causing temperatures to rise.
    3. The adjustments should be applied only to the coldest areas, not to every area below 0 degrees.

    To achieve this, we'll modify the code in the plot_stations_with_boundary_and_interpolation function where the ECOSTRESS adjustments are applied. We'll:

    • Replace the current day_weight function with a new night_weight function that peaks at midnight.
    • Define the coldest areas based on the ECOSTRESS anomalies.
    • Apply adjustments only to these coldest areas and modulate it with the night_weight.

    Here's the updated code:

    def plot_stations_with_boundary_and_interpolation(
            trax_stations, trax_temperatures, trax_coords,
            netatmo_stations, netatmo_temperatures, netatmo_coords,
            imgw_stations, imgw_temperatures, imgw_coords):
        # [ ... existing code up to ECOSTRESS Adjustments Begin Here ... ]
    
        # ECOSTRESS Adjustments Begin Here
    
        # Retrieve ECOSTRESS LST values at grid points (same as before)
        grid_ecostress = get_raster_value_at_points(
            [Point(x, y) for x, y in zip(grid_df['X'], grid_df['Y'])],
            raster_data=ecostress_data,
            raster_transform=ecostress_transform
        )
        grid_df['ECOSTRESS_LST'] = grid_ecostress
    
        # Handle NaNs in ECOSTRESS_LST by filling with the mean value
        grid_df['ECOSTRESS_LST'].fillna(grid_df['ECOSTRESS_LST'].mean(), inplace=True)
    
        # Compute ECOSTRESS anomalies (deviation from mean)
        ecostress_mean = grid_df['ECOSTRESS_LST'].mean()
        grid_df['ECOSTRESS_Anomaly'] = grid_df['ECOSTRESS_LST'] - ecostress_mean
    
        # Normalize anomalies to range [-1, 1]
        ecostress_anomaly_min = grid_df['ECOSTRESS_Anomaly'].min()
        ecostress_anomaly_max = grid_df['ECOSTRESS_Anomaly'].max()
        grid_df['ECOSTRESS_Anomaly_Normalized'] = (
            (grid_df['ECOSTRESS_Anomaly'] - ecostress_anomaly_min) / (ecostress_anomaly_max - ecostress_anomaly_min) * 2 - 1
        )
    
        # Increase the impact of ECOSTRESS LST data
        amplification_factor = 3.0  # Adjust as needed
    
        # Determine the current time in CET
        cet = pytz.timezone('CET')
        now_cet = datetime.now(cet)
        hour = now_cet.hour + now_cet.minute / 60.0
    
        # Compute hour angle for cosine function
        hour_angle = ((hour % 24) / 24) * 2 * math.pi  # Convert hour to radians
    
        # Compute night_weight: maximum at midnight, zero at noon
        night_weight = (math.cos(hour_angle) + 1) / 2  # Ranges from 1 at midnight to 0 at noon
    
        # Define threshold for coldest areas (e.g., where ECOSTRESS anomaly is significantly low)
        coldest_threshold = -0.5  # You can adjust this value
        coldest_areas_mask = grid_df['ECOSTRESS_Anomaly_Normalized'] <= coldest_threshold
    
        # Apply anomalies with the new night_weight and only to coldest areas
        adjustment = grid_df['ECOSTRESS_Anomaly_Normalized'] * amplification_factor * night_weight
    
        # Apply adjustments only in the coldest areas
        adjustment[~coldest_areas_mask] = 0
    
        # Prevent ECOSTRESS adjustments on water bodies
        adjustment[grid_df['Water_Body'] == 1] = 0
    
        # Limit maximum adjustment to avoid over-adjusting
        max_adjustment = 5.0  # degrees Celsius
        adjustment = adjustment.clip(-max_adjustment, max_adjustment)
    
        # Apply adjustments to the interpolated temperatures
        grid_df['Temperature'] += adjustment
    
        # Apply cooling effect on water bodies
        cooling_effect = -2.0  # degrees Celsius
        grid_df.loc[grid_df['Water_Body'] == 1, 'Temperature'] += cooling_effect
    
        # [ ... rest of the code remains unchanged ... ]
    

    Explanation:

    • Night Weight Calculation: We replaced the day_weight with a night_weight that peaks at midnight (hour 0) and decreases smoothly until noon (hour 12), then increases again towards midnight. This is achieved using the cosine function:

      night_weight = (math.cos(hour_angle) + 1) / 2
      

      This function gives a value between 0 and 1, with 1 at midnight and 0 at noon.

    • Defining Coldest Areas: We identify the coldest areas based on the ECOSTRESS_Anomaly_Normalized. For this example, we set a threshold of -0.5, meaning only areas with significantly negative anomalies are considered:

      coldest_threshold = -0.5  # Adjust this value as needed
      coldest_areas_mask = grid_df['ECOSTRESS_Anomaly_Normalized'] <= coldest_threshold
      
    • Applying Adjustments:

      • We apply the adjustments only to the coldest areas by masking out other areas:
        adjustment[~coldest_areas_mask] = 0
        
      • The adjustments are modulated by the night_weight, ensuring they are stronger during the deep night and decrease towards sunrise.
      • We prevent adjustments on water bodies and limit the maximum adjustment as before.
    • Smooth Temperature Rise After Deep Night: As the night_weight decreases smoothly after midnight towards sunrise, the negative adjustments become smaller, causing temperatures in the coldest areas to rise smoothly.

    Note:

    • You might need to adjust coldest_threshold or use percentiles based on your data to accurately capture the coldest areas.
    • Ensure that the rest of your code (e.g., plotting functions) remains consistent with these changes.

    Testing the Changes:

    Make sure to test the modified script during different times of the day to verify that:

    • During the night, the coldest areas receive additional cooling.
    • After the deep night, the adjustments decrease, leading to a gradual temperature rise.
    • Adjustments are only applied to the coldest areas, not uniformly across all areas below 0 degrees.

    Example Output:

    After running the script with these modifications, you should observe that:

    • The Temperature Interpolation with ECOSTRESS Adjustments plot shows colder temperatures in the coldest areas during the night compared to the version without adjustments.
    • The Difference Due to ECOSTRESS Adjustments plot highlights the areas where adjustments have been applied, showing larger negative values in the coldest areas during the night.

    By implementing these modifications, your script should now adjust the temperatures based on the ECOSTRESS data, with the adjustments varying depending on the time of day and focusing on the coldest areas as per your requirements.