import numpy as np import matplotlib.pyplot as plt import rasterio from…
- User
import numpy as np import matplotlib.pyplot as plt import rasterio from rasterio.transform import from_origin from rasterio.warp import calculate_default_transform, reproject, Resampling import geopandas as gpd from shapely.geometry import Point, Polygon from pyproj import Proj, Transformer import struct from tqdm import tqdm import os from lxml import etree import io import sys import logging
Configure logging logging.basicConfig( level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s', handlers=[ logging.StreamHandler(sys.stdout), logging.FileHandler('radar_data_processing.log', mode='w') ] )
Define constants and paths RADAR_DATA_PATH = 'input/radar/dBZ.vol' # Path to the radar data file MASOVIAN_BOUNDARY_PATH = 'input/radar/masovian_boundary.geojson' # Path to Masovian boundary GeoJSON or Shapefile OUTPUT_PLOT_PATH = 'output/processed_radar_data.png' # Path to save the output plot
Radar specifications based on provided XML AZIMUTH_RESOLUTION = 1.0 # Degrees between each azimuth RANGE_RESOLUTION = 0.5 # As per 0.5 km
Radar location from RADAR_LATITUDE = 52.405250 # From 52.405250 RADAR_LONGITUDE = 20.961110 # From 20.961110 RADAR_ALTITUDE = 122.3 # From 122.300000 meters
Coordinate Systems SOURCE_CRS = 'EPSG:4326' # Geographic coordinates TARGET_CRS = 'EPSG:2180' # Polish coordinate system
def parse_vol_file(file_path): """ Parses the .vol file to extract metadata about each radar slice.
Parameters:
- file_path: Path to the .vol file.
Returns:
- slices_info: List of dictionaries containing slice metadata.
- sensor_info: Dictionary containing radar sensor information.
- binary_data_start: Byte index where binary data starts. """ logging.info("Parsing the .vol file...") slices_info = [] sensor_info = {}
with open(file_path, 'rb') as f: # Read the entire file content = f.read()
Locate the end of the XML content
end_volume_tag = b'' end_volume_index = content.find(end_volume_tag) if end_volume_index == -1: logging.error(" tag not found in the file.") sys.exit(1)
xml_content = content[:end_volume_index + len(end_volume_tag)] binary_content = content[end_volume_index + len(end_volume_tag):]
Parse XML
try: root = etree.fromstring(xml_content) except etree.XMLSyntaxError as e: logging.error(f"Failed to parse XML content: {e}") sys.exit(1)
Extract sensor info
sensor = root.find('.//sensorinfo') if sensor is not None: sensor_info['id'] = sensor.findtext('id') sensor_info['type'] = sensor.findtext('type') sensor_info['name'] = sensor.findtext('name') try: sensor_info['lon'] = float(sensor.findtext('lon')) sensor_info['lat'] = float(sensor.findtext('lat')) sensor_info['alt'] = float(sensor.findtext('alt')) sensor_info['wavelen'] = float(sensor.findtext('wavelen')) sensor_info['beamwidth'] = float(sensor.findtext('beamwidth')) except (TypeError, ValueError) as e: logging.error(f"Invalid sensor information: {e}") sys.exit(1) else: logging.error(" not found in the XML.") sys.exit(1)
Extract slice information
slices = root.findall('.//slice') for slice_elem in slices: slice_info = {} slice_info['refid'] = slice_elem.get('refid') # Find rawdata element for dBZ rawdata = slice_elem.find('.//rawdata[@type="dBZ"]') if rawdata is not None: try: slice_info['rays'] = int(rawdata.get('rays')) slice_info['bins'] = int(rawdata.get('bins')) slice_info['type'] = rawdata.get('type') slice_info['max'] = float(rawdata.get('max')) slice_info['min'] = float(rawdata.get('min')) slices_info.append(slice_info) logging.info(f"Slice refid {slice_info['refid']}: rays {slice_info['rays']}, bins {slice_info['bins']}.") except (TypeError, ValueError) as e: logging.warning(f"Invalid slice information in refid {slice_info.get('refid')}: {e}. Skipping this slice.") else: logging.warning(f"No dBZ rawdata found for slice refid {slice_elem.get('refid')}. Skipping this slice.")
Determine where binary data starts
binary_data_start = end_volume_index + len(end_volume_tag)
logging.info(f"Total slices with dBZ data: {len(slices_info)}.\n")
return slices_info, sensor_info, binary_data_start, binary_content def extract_dBZ_data(slices_info, binary_content, binary_data_start): """ Extracts dBZ data from the binary content based on slice metadata.
Parameters:
- slices_info: List of dictionaries containing slice metadata.
- binary_content: Bytes object containing all binary data after .
- binary_data_start: Byte index where binary data starts.
Returns:
- dBZ_combined: 2D numpy array combining all slices (azimuths x range_bins). """ logging.info("Extracting dBZ data from binary content...") dBZ_list = [] offset = 0 # Current byte offset within binary_content
for idx, slice_info in enumerate(slices_info): rays = slice_info['rays'] bins = slice_info['bins'] dtype = np.float32 # float32 per dBZ value byte_size = 4 # bytes per float32 expected_bytes = rays * bins * byte_size
# Check if enough data remains if offset + expected_bytes > len(binary_content): logging.warning(f"Slice {idx + 1} refid {slice_info['refid']}: Insufficient binary data. Expected {expected_bytes} bytes, got {len(binary_content) - offset} bytes. Adjusting number of bins.") # Adjust the number of bins based on remaining data available_bytes = len(binary_content) - offset valid_bins = available_bytes // (rays * byte_size) if valid_bins <= 0: logging.error(f"Slice {idx + 1} refid {slice_info['refid']}: No data available. Skipping this slice.") continue slice_info['bins'] = valid_bins bins = valid_bins expected_bytes = rays * bins * byte_size # Extract binary data for this slice slice_binary_data = binary_content[offset:offset + expected_bytes] offset += expected_bytes # Move to the next slice try: # Convert binary data to numpy array dBZ = np.frombuffer(slice_binary_data, dtype=dtype).reshape((rays, bins)) dBZ_list.append(dBZ) logging.info(f"Slice {idx + 1} refid {slice_info['refid']}: Extracted dBZ data with shape {dBZ.shape}.") except Exception as e: logging.error(f"Slice {idx + 1} refid {slice_info['refid']}: Failed to parse dBZ data. Error: {e}. Skipping this slice.") continueif not dBZ_list: logging.error("No dBZ data extracted. Exiting.") sys.exit(1)
Ensure all slices have the same number of bins by padding with np.nan
max_bins = max(dBZ.shape[1] for dBZ in dBZ_list) logging.info(f"Maximum number of bins across slices: {max_bins}")
padded_dBZ_list = [] for idx, dBZ in enumerate(dBZ_list): current_bins = dBZ.shape[1] if current_bins < max_bins: padding = ((0, 0), (0, max_bins - current_bins)) dBZ_padded = np.pad(dBZ, padding, mode='constant', constant_values=np.nan) logging.info(f"Padded slice {idx + 1} from {current_bins} to {max_bins} bins.") else: dBZ_padded = dBZ padded_dBZ_list.append(dBZ_padded)
Stack all slices vertically
try: dBZ_combined = np.vstack(padded_dBZ_list) logging.info(f"Combined dBZ data shape after padding and stacking: {dBZ_combined.shape}\n") except Exception as e: logging.error(f"Failed to stack dBZ data: {e}") sys.exit(1)
return dBZ_combined def polar_to_cartesian(dBZ, radar_lat, radar_lon, az_res, rng_res): """ Converts polar radar data to Cartesian coordinates.
Parameters:
- dBZ: 2D numpy array of dBZ values (azimuths x ranges).
- radar_lat: Latitude of the radar.
- radar_lon: Longitude of the radar.
- az_res: Azimuth resolution in degrees.
- rng_res: Range resolution in km.
Returns:
- xs: 1D numpy array of x coordinates in meters.
- ys: 1D numpy array of y coordinates in meters.
- dBZ_1D: 1D numpy array of dBZ values. """ logging.info("Converting polar coordinates to Cartesian coordinates...") num_az, num_rng = dBZ.shape azimuths = np.deg2rad(np.arange(0, 360, az_res)) ranges = np.arange(rng_res, rng_res * (num_rng + 1), rng_res) * 1000 # Convert km to meters
Ensure azimuths match the number of rays
if len(azimuths) != num_az: logging.warning(f"Number of azimuths ({len(azimuths)}) does not match expected ({num_az}). Adjusting.") azimuths = np.linspace(0, 2 * np.pi, num=num_az, endpoint=False)
Adjust ranges if num_rng does not match expected
if len(ranges) != num_rng: logging.warning(f"Number of range bins ({len(ranges)}) does not match expected ({num_rng}). Adjusting.") ranges = np.linspace(rng_res, rng_res * num_rng, num=num_rng) * 1000 # Ensure in meters
Create meshgrid
az, r = np.meshgrid(azimuths, ranges, indexing='ij')
Convert to Cartesian coordinates
x = r * np.sin(az) y = r * np.cos(az)
Flatten the arrays
xs = x.flatten() ys = y.flatten() dBZ_1D = dBZ.flatten()
logging.info("Coordinate conversion completed.\n") return xs, ys, dBZ_1D def create_geodataframe(xs, ys, dBZ): """ Creates a GeoDataFrame from Cartesian coordinates and dBZ values.
Parameters:
- xs: 1D numpy array of x coordinates in meters.
- ys: 1D numpy array of y coordinates in meters.
- dBZ: 1D numpy array of dBZ values.
Returns:
- gdf: GeoDataFrame with geometry in geographic coordinates. """ logging.info("Creating GeoDataFrame...")
Define local projection (using Transverse Mercator centered at radar location)
proj_local = Proj(proj='tmerc', lat_0=RADAR_LATITUDE, lon_0=RADAR_LONGITUDE, ellps='WGS84') transformer = Transformer.from_proj(proj_local, 'epsg:4326', always_xy=True)
Transform to geographic coordinates
lon, lat = transformer.transform(xs, ys)
Create GeoDataFrame
geometry = [Point(xy) for xy in zip(lon, lat)] gdf = gpd.GeoDataFrame({'dBZ': dBZ}, geometry=geometry, crs='EPSG:4326')
logging.info("GeoDataFrame creation completed.\n") return gdf def reproject_gdf(gdf, target_crs='EPSG:2180'): """ Reprojects the GeoDataFrame to the target CRS.
Parameters:
- gdf: Input GeoDataFrame.
- target_crs: Target Coordinate Reference System.
Returns:
- gdf_reprojected: Reprojected GeoDataFrame. """ logging.info(f"Reprojecting GeoDataFrame to {target_crs}...") try: gdf_reprojected = gdf.to_crs(target_crs) logging.info("Reprojection completed.\n") except Exception as e: logging.error(f"Failed to reproject GeoDataFrame: {e}") sys.exit(1) return gdf_reprojected def clip_to_masovian(gdf, masovian_boundary): """ Clips the GeoDataFrame to the Masovian boundary.
Parameters:
- gdf: Input GeoDataFrame.
- masovian_boundary: GeoDataFrame or Polygon of Masovian boundary.
Returns:
- gdf_clipped: Clipped GeoDataFrame.
"""
logging.info("Clipping radar data to Masovian boundary...")
try:
Spatial join or intersection
gdf_clipped = gdf[gdf.within(masovian_boundary.unary_union)] logging.info(f"Clipped data contains {len(gdf_clipped)} points.\n") except Exception as e: logging.error(f"Failed to clip data to Masovian boundary: {e}") sys.exit(1) return gdf_clipped def visualize_radar_data(gdf, masovian_boundary, output_path): """ Visualizes the radar data.
Parameters:
- gdf: GeoDataFrame containing clipped radar data.
- masovian_boundary: GeoDataFrame or Polygon of Masovian boundary.
- output_path: Path to save the plot.
Returns:
-
None """ logging.info("Visualizing radar data...") try: fig, ax = plt.subplots(figsize=(12, 12))
Plot Masovian boundary
masovian_boundary.plot(ax=ax, edgecolor='black', facecolor='none', linewidth=1)
Plot radar dBZ values using scatter plot
Filter out invalid dBZ values (e.g., NaN or extreme values)
valid_gdf = gdf[(gdf['dBZ'] >= 0) & (gdf['dBZ'] <= 80)]
scatter = valid_gdf.plot( ax=ax, column='dBZ', cmap='jet', markersize=1, alpha=0.6, legend=True, legend_kwds={'label': "dBZ Value", 'shrink': 0.5} )
ax.set_title("Processed Radar dBZ Data within Masovian Region", fontsize=16) ax.set_xlabel("Easting (meters)", fontsize=12) ax.set_ylabel("Northing (meters)", fontsize=12) ax.set_aspect('equal')
Remove axis ticks for cleaner look
ax.set_xticks([]) ax.set_yticks([])
Add colorbar
sm = plt.cm.ScalarMappable(cmap='jet', norm=plt.Normalize(vmin=valid_gdf['dBZ'].min(), vmax=valid_gdf['dBZ'].max())) sm._A = [] # dummy array for the scalar mappable cbar = fig.colorbar(sm, ax=ax, fraction=0.036, pad=0.04) cbar.set_label('dBZ', fontsize=12)
Save the plot
plt.savefig(output_path, dpi=300, bbox_inches='tight') plt.close() logging.info(f"Radar data visualization saved to {output_path}.\n") except Exception as e: logging.error(f"Failed to visualize radar data: {e}") sys.exit(1) def main():
Step 1: Parse the .vol file
if not os.path.exists(RADAR_DATA_PATH): logging.error(f"Radar data file not found at {RADAR_DATA_PATH}. Please check the path.") sys.exit(1)
slices_info, sensor_info, binary_data_start, binary_content = parse_vol_file(RADAR_DATA_PATH)
Step 2: Extract dBZ data from binary content
dBZ_combined = extract_dBZ_data(slices_info, binary_content, binary_data_start)
Step 3: Convert polar to Cartesian coordinates
xs, ys, dBZ_1D = polar_to_cartesian( dBZ_combined, RADAR_LATITUDE, RADAR_LONGITUDE, AZIMUTH_RESOLUTION, RANGE_RESOLUTION )
Step 4: Create GeoDataFrame
gdf_geo = create_geodataframe(xs, ys, dBZ_1D)
Step 5: Reproject to EPSG:2180
gdf_proj = reproject_gdf(gdf_geo, TARGET_CRS)
Step 6: Load Masovian Boundary
if not os.path.exists(MASOVIAN_BOUNDARY_PATH): logging.error(f"Masovian boundary file not found at {MASOVIAN_BOUNDARY_PATH}. Please check the path.") sys.exit(1) try: masovian_boundary = gpd.read_file(MASOVIAN_BOUNDARY_PATH) logging.info("Loaded Masovian boundary successfully.\n") except Exception as e: logging.error(f"Failed to load Masovian boundary: {e}") sys.exit(1)
Ensure Masovian boundary is in the same CRS as target CRS
if masovian_boundary.crs != TARGET_CRS: try: masovian_boundary = masovian_boundary.to_crs(TARGET_CRS) logging.info(f"Reprojected Masovian boundary to {TARGET_CRS}.\n") except Exception as e: logging.error(f"Failed to reproject Masovian boundary: {e}") sys.exit(1)
Step 7: Clip Radar Data to Masovian Boundary
gdf_clipped = clip_to_masovian(gdf_proj, masovian_boundary)
Step 8: Visualize and Plot
visualize_radar_data(gdf_clipped, masovian_boundary, OUTPUT_PLOT_PATH)
logging.info("Radar data processing and visualization completed successfully.") if name == "main": main() dBZ.vol:
SI 0 1 249 2024-10-05T18:46:32 2024-10-05T18:49:14 2024-10-05T18:46:30 2024-10-05T18:51:30
Off 2 3 Off Off 2 0 250 8 Optimised 0.5 40 0 Off GdrxDpModeHV_HV 550.00 None 0.00 0.00 2.05 FreePRF 0 0.5 1 Off 24 1 23 None 0 Off On FFT On Off Off Off Off NoAlgType 6 0.5000 On GdrxCmOff 1 Interpolate 18 18 On AngleStep false 30 0.3 0.5 2
SdfDPrfModeAda 2 On SdfDPrfCorrMedian SdfCfDepthDefault GdrxWfHamming SdfSpatialFilterAvg 3 5 0.5 20 3 0.4 GdrxWfRectangular CCOR,CCORv,dBZ,dBuZ,dBZv,dBuZv,FLAGS,uKDP,KDP,uPhiDP,PhiDP,RhoHV,SNR,SNRu,SNRv,SQI,SQIv,V,W,ZDR CCOR,dBZ,FLAGS,KDP,PhiDP,RhoHV,SNR,SQI,V,W,ZDR dBZ=0,dBZv=0,dBuZ=0,dBuZv=0,V=0,Vv=0,Vu=0,Vvu=0,W=0,Wv=0,Wu=0,Wvu=0,ZDR=0,ZDRu=0,uPhiDP=1,uPhiDPu=1,PhiDP=1,KDP=0,uKDP=0,uKDPu=0,RhoHV=0,RhoHVu=0,CCOR=0,CCORv=0,SNR=1,SNRv=0,SNRu=0,SNRvu=0,SQI=0,SQIu=0,SQIv=0,SQIvu=0,FLAGS=-1 23.800000,15.000000,9.100000,5.400000,3.500000,2.500000,1.500000,0.500000,3.500000,2.500000,1.500000,0.500000 0.50 Off 0.00 Off
0.5 18 29 DATA 0.4 3 10 10 0 None dBZ:622602,dBZv:622602,dBuZ:1,dBuZv:1,V:524294,Vv:524294,Vu:1,Vvu:1,W:524294,Wv:524294,Wu:1,Wvu:1,ZDR:622602,ZDRu:1,uPhiDP:2,uPhiDPu:1,PhiDP:524290,KDP:524290,uKDP:2,uKDPu:1,RhoHV:622594,RhoHVu:1 2 3 125 0 0.5 4 Off Off GdrxWfHamming SdfSpatialFilterAvg 3 5 0.0 0.0 1200 4/3 SdfDPrfCorrMedian On 2 SdfDPrfModeAda 900.00 Off 4 SdfCfDepthDefault Off Off Off 0 AngleStep 1 0 On 30 0.1 0.1 2 On
2 GdrxCmOff 0.5 20 3 0.4 GdrxWfRectangular Off On On 0.5000 On On Interpolate Off On Off 0.00 SdfStaticLookupFilter 0.95 On 80 None FreePRF 3.00 8 12 28462 6139 262plc1_sq 262plc1_sq_lk OK OK 5649.8447 OK 93.3 0 0.019 5.31 122.3 OK 45.38 20.7 217.77 45.37 21.08 3.51 0.0 210.86 0.66 0.95 0.00 0.2 0.93 3.32 0.0 217.77 0.66 0.99 OK OK 1100000 -0.15 -42.2651 -42.3474 69.534 69.534 69.884 4 20 0.0 59.8483
1.5 7 6138 59.9227
2.5 6 6137 60.0078
3.5 5 6136 -42.3475 60.0972
5.4 170 882 661.50 1.29 16 6 6 24 0.3 4 6135 800000 -44.9218 -45.0041 68.206 68.206 68.556 60.2026
9.1 20 26 DATA,NOISE 0.3 3 6134 60.2829
15.0 24 DATA,NOISE 22 0.3 2 16 16 2 6133 60.111
23.8 0.3 2 16 16 22 24 DATA,NOISE 1 6132 -42.3475 59.9191
20.961110 52.405250 122.300000 0.0531 0.95
So, basically, here I'm attemping to create a Python script that will perform next tasks:
Read RAINBOW volume dBZ radar data (path to the data file: dBZ.vol) Reproject the radar data file from original raw polar coordinates that are in the file to cartesian coordinates, and later to EPSG:2180 coordinate system Clip the radar data to the extent of Masovian region Visualize and plot the processed radar data in a simple way Though currently, unfortunately, I'm encountering with several issues in the Python code. Your task - is to fix all of them and provide a complete updated code. The issues are: Color bar on the plot is being duplicated The extent of radar data on the plot is way smaller than it should be (the original raw extent of the radar scanning is perfectly 125km from the radar site, a circle-like extent common for the weather radars + of course this should be accounted to the data cutting for specific masovian region extent), here's a basic function that might help with getting the masovian region extent: def get_masovian_boundary(): url = "https://raw.githubusercontent.com/ppatrzyk/polska-geojson/master/wojewodztwa/wojewodztwa-medium.geojson" Fetch the content using requests response = requests.get(url) if response.status_code == 200: data = response.json()
Set the CRS to EPSG:4326 when creating the GeoDataFrame
gdf = gpd.GeoDataFrame.from_features(data["features"], crs="EPSG:4326") masovian = gdf[gdf['nazwa'] == 'mazowieckie'] return masovian else: print(f"Error fetching GeoJSON data: {response.status_code}") return None But of course in our code we already have a data file - the masovian_boundary.geojson. But it's structure is literally the same 1:1. 3. The radar data plotted on the plot differs from my expectations. It looks like a small circle filled with dark blue color and random, grainy data not representing actual radar data storaged in the dBZ.vol that we need to decode and visualize
Here's also a console log returning when running the code that might significantly help you: 2024-10-05 21:36:01,864 - INFO - Parsing the .vol file... 2024-10-05 21:36:01,866 - INFO - Slice refid 0: rays 360, bins 250. 2024-10-05 21:36:01,866 - INFO - Slice refid 1: rays 360, bins 250. 2024-10-05 21:36:01,866 - INFO - Slice refid 2: rays 360, bins 250. 2024-10-05 21:36:01,867 - INFO - Slice refid 3: rays 360, bins 250. 2024-10-05 21:36:01,867 - INFO - Slice refid 4: rays 360, bins 340. 2024-10-05 21:36:01,867 - INFO - Slice refid 5: rays 360, bins 250. 2024-10-05 21:36:01,867 - INFO - Slice refid 6: rays 360, bins 250. 2024-10-05 21:36:01,868 - INFO - Slice refid 7: rays 360, bins 250. 2024-10-05 21:36:01,868 - INFO - Total slices with dBZ data: 8.
2024-10-05 21:36:01,868 - INFO - Extracting dBZ data from binary content... 2024-10-05 21:36:01,869 - WARNING - Slice 1 refid 0: Insufficient binary data. Expected 360000 bytes, got 182926 bytes. Adjusting number of bins. 2024-10-05 21:36:01,869 - INFO - Slice 1 refid 0: Extracted dBZ data with shape (360, 127). 2024-10-05 21:36:01,869 - WARNING - Slice 2 refid 1: Insufficient binary data. Expected 360000 bytes, got 46 bytes. Adjusting number of bins. 2024-10-05 21:36:01,869 - ERROR - Slice 2 refid 1: No data available. Skipping this slice. 2024-10-05 21:36:01,869 - WARNING - Slice 3 refid 2: Insufficient binary data. Expected 360000 bytes, got 46 bytes. Adjusting number of bins. 2024-10-05 21:36:01,869 - ERROR - Slice 3 refid 2: No data available. Skipping this slice. 2024-10-05 21:36:01,869 - WARNING - Slice 4 refid 3: Insufficient binary data. Expected 360000 bytes, got 46 bytes. Adjusting number of bins. 2024-10-05 21:36:01,870 - ERROR - Slice 4 refid 3: No data available. Skipping this slice. 2024-10-05 21:36:01,870 - WARNING - Slice 5 refid 4: Insufficient binary data. Expected 489600 bytes, got 46 bytes. Adjusting number of bins. 2024-10-05 21:36:01,870 - ERROR - Slice 5 refid 4: No data available. Skipping this slice. 2024-10-05 21:36:01,870 - WARNING - Slice 6 refid 5: Insufficient binary data. Expected 360000 bytes, got 46 bytes. Adjusting number of bins. 2024-10-05 21:36:01,871 - ERROR - Slice 6 refid 5: No data available. Skipping this slice. 2024-10-05 21:36:01,871 - WARNING - Slice 7 refid 6: Insufficient binary data. Expected 360000 bytes, got 46 bytes. Adjusting number of bins. 2024-10-05 21:36:01,871 - ERROR - Slice 7 refid 6: No data available. Skipping this slice. 2024-10-05 21:36:01,871 - WARNING - Slice 8 refid 7: Insufficient binary data. Expected 360000 bytes, got 46 bytes. Adjusting number of bins. 2024-10-05 21:36:01,871 - ERROR - Slice 8 refid 7: No data available. Skipping this slice. 2024-10-05 21:36:01,872 - INFO - Maximum number of bins across slices: 127 2024-10-05 21:36:01,872 - INFO - Combined dBZ data shape after padding and stacking: (360, 127)
2024-10-05 21:36:01,872 - INFO - Converting polar coordinates to Cartesian coordinates... 2024-10-05 21:36:01,874 - INFO - Coordinate conversion completed.
2024-10-05 21:36:01,874 - INFO - Creating GeoDataFrame... 2024-10-05 21:36:03,113 - INFO - GeoDataFrame creation completed.
2024-10-05 21:36:03,120 - INFO - Reprojecting GeoDataFrame to EPSG:2180... 2024-10-05 21:36:04,891 - INFO - Reprojection completed.
2024-10-05 21:36:05,452 - INFO - Loaded Masovian boundary successfully.
2024-10-05 21:36:05,560 - INFO - Reprojected Masovian boundary to EPSG:2180.
2024-10-05 21:36:05,576 - INFO - Clipping radar data to Masovian boundary... 2024-10-05 21:39:18,682 - INFO - Clipped data contains 45720 points.
2024-10-05 21:39:18,682 - INFO - Visualizing radar data... 2024-10-05 21:39:20,075 - INFO - Radar data visualization saved to output/processed_radar_data.png.
2024-10-05 21:39:20,076 - INFO - Radar data processing and visualization completed successfully.
- Assistant
It looks like you're encountering several issues because the format of your radar data file (
dBZ.vol) doesn't match the format expected by the script. Let's address each issue step by step and adjust the code accordingly.Issue 1: The Radar Data File Format
Your script expects the radar data file to be a Rainbow
.volfile containing XML metadata followed by binary data. However, the content you've provided fordBZ.volappears to be in a different format—it's a text file without any XML structure or binary data.Solution: Adjust the Code to Match the Data Format
Since your
dBZ.volfile doesn't contain XML or binary data, we need to adjust the code to parse the text data correctly. We'll modify theparse_vol_filefunction to read the file line by line and extract the necessary information.Here's how you can adjust your script:
-
Modify the
parse_vol_fileFunction:We'll change this function to read the file as a text file and parse the elevation angles, number of rays, number of bins, and other relevant metadata. Since the data seems to be in a custom text format, we'll use regular expressions or string parsing methods to extract the data.
-
Extract dBZ Data Correctly:
We'll adjust the
extract_dBZ_datafunction to read the dBZ values from the text data instead of expecting binary data. We'll need to parse the lines where the dBZ values are located and convert them into a numerical array. -
Ensure Correct Azimuth and Range Resolutions:
We'll verify and adjust the azimuth and range resolutions based on the data provided in your
dBZ.volfile. -
Fix the Visualization Issues:
By correctly parsing the data, we should be able to create a more accurate visualization. We can also adjust the plotting code to ensure the color bar is displayed correctly and that the extent of the radar data matches expectations.
Let's proceed to modify your code.
Updated Code
import numpy as np import matplotlib.pyplot as plt import geopandas as gpd from shapely.geometry import Point from pyproj import Proj, Transformer import logging import sys import os # Configure logging logging.basicConfig( level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s', handlers=[ logging.StreamHandler(sys.stdout), logging.FileHandler('radar_data_processing.log', mode='w') ] ) # Define constants and paths RADAR_DATA_PATH = 'input/radar/dBZ.vol' # Path to the radar data file MASOVIAN_BOUNDARY_PATH = 'input/radar/masovian_boundary.geojson' # Path to Masovian boundary GeoJSON or Shapefile OUTPUT_PLOT_PATH = 'output/processed_radar_data.png' # Path to save the output plot # Radar location RADAR_LATITUDE = 52.405250 RADAR_LONGITUDE = 20.961110 RADAR_ALTITUDE = 122.3 # meters # Coordinate Systems SOURCE_CRS = 'EPSG:4326' # Geographic coordinates TARGET_CRS = 'EPSG:2180' # Polish coordinate system def parse_vol_file(file_path): """ Parses the .vol file to extract metadata and dBZ data. """ logging.info("Parsing the .vol file...") slices_info = [] sensor_info = {} dBZ_data = [] elevations = [] try: with open(file_path, 'r') as f: lines = f.readlines() # Extract metadata and dBZ data for i, line in enumerate(lines): line = line.strip() if line == '': continue # Extract radar location info if line == '20.961110' and i + 2 < len(lines): sensor_info['lon'] = float(lines[i]) sensor_info['lat'] = float(lines[i+1]) sensor_info['alt'] = float(lines[i+2]) # Extract elevations try: elevation = float(line) # We assume that an elevation angle is followed by other data if elevation >= 0 and elevation <= 90: elevations.append(elevation) logging.info(f"Found elevation angle: {elevation}") except ValueError: pass # Extract dBZ values if 'dBZ:' in line: parts = line.split(',') for part in parts: if part.startswith('dBZ:'): dBZ_value = part.split(':')[1] try: dBZ_value = float(dBZ_value) dBZ_data.append(dBZ_value) except ValueError: pass if not dBZ_data: logging.error("No dBZ data found in the file.") sys.exit(1) logging.info(f"Extracted {len(dBZ_data)} dBZ values.\n") # Assuming we have elevations and dBZ data return dBZ_data, elevations, sensor_info except Exception as e: logging.error(f"Failed to parse the .vol file: {e}") sys.exit(1) def create_polar_grid(elevations, radar_range_km, azimuths): """ Creates a polar grid based on elevations, range, and azimuth angles. """ logging.info("Creating polar grid...") r = np.linspace(0, radar_range_km * 1000, num=radar_range_km*2) # Assuming 0.5 km range bins az = np.deg2rad(azimuths) az_grid, r_grid = np.meshgrid(az, r, indexing='ij') return az_grid, r_grid def polar_to_cartesian(az_grid, r_grid): """ Converts polar coordinates to Cartesian coordinates. """ logging.info("Converting polar coordinates to Cartesian coordinates...") x = r_grid * np.sin(az_grid) y = r_grid * np.cos(az_grid) return x, y def create_geodataframe(xs, ys, dBZ_values): """ Creates a GeoDataFrame from Cartesian coordinates and dBZ values. """ logging.info("Creating GeoDataFrame...") proj_local = Proj(proj='aeqd', lat_0=RADAR_LATITUDE, lon_0=RADAR_LONGITUDE, datum='WGS84') transformer = Transformer.from_proj(proj_local, 'epsg:4326', always_xy=True) lon, lat = transformer.transform(xs, ys) # Flatten arrays lon = lon.flatten() lat = lat.flatten() dBZ_values = np.array(dBZ_values).flatten() # Create GeoDataFrame geometry = [Point(xy) for xy in zip(lon, lat)] gdf = gpd.GeoDataFrame({'dBZ': dBZ_values}, geometry=geometry, crs='EPSG:4326') logging.info("GeoDataFrame creation completed.\n") return gdf def main(): # Step 1: Parse the .vol file if not os.path.exists(RADAR_DATA_PATH): logging.error(f"Radar data file not found at {RADAR_DATA_PATH}. Please check the path.") sys.exit(1) dBZ_data, elevations, sensor_info = parse_vol_file(RADAR_DATA_PATH) # Use the radar location from the file if available if 'lat' in sensor_info: radar_lat = sensor_info['lat'] else: radar_lat = RADAR_LATITUDE if 'lon' in sensor_info: radar_lon = sensor_info['lon'] else: radar_lon = RADAR_LONGITUDE # For simplicity, we'll assume azimuths from 0 to 359 degrees azimuths = np.arange(0, 360, 1) # 1-degree resolution # Assume radar range is 125 km radar_range_km = 125 # Step 2: Create polar grid az_grid, r_grid = create_polar_grid(elevations, radar_range_km, azimuths) # Step 3: Convert polar to Cartesian coordinates xs, ys = polar_to_cartesian(az_grid, r_grid) # Step 4: Create GeoDataFrame gdf_geo = create_geodataframe(xs, ys, dBZ_data) # Step 5: Reproject to EPSG:2180 try: gdf_proj = gdf_geo.to_crs(TARGET_CRS) logging.info(f"Reprojected GeoDataFrame to {TARGET_CRS}.\n") except Exception as e: logging.error(f"Failed to reproject GeoDataFrame: {e}") sys.exit(1) # Step 6: Load Masovian Boundary if not os.path.exists(MASOVIAN_BOUNDARY_PATH): logging.error(f"Masovian boundary file not found at {MASOVIAN_BOUNDARY_PATH}. Please check the path.") sys.exit(1) try: masovian_boundary = gpd.read_file(MASOVIAN_BOUNDARY_PATH) logging.info("Loaded Masovian boundary successfully.\n") except Exception as e: logging.error(f"Failed to load Masovian boundary: {e}") sys.exit(1) # Ensure Masovian boundary is in the same CRS as target CRS if masovian_boundary.crs != TARGET_CRS: try: masovian_boundary = masovian_boundary.to_crs(TARGET_CRS) logging.info(f"Reprojected Masovian boundary to {TARGET_CRS}.\n") except Exception as e: logging.error(f"Failed to reproject Masovian boundary: {e}") sys.exit(1) # Step 7: Clip Radar Data to Masovian Boundary try: gdf_clipped = gpd.sjoin(gdf_proj, masovian_boundary, how='inner', op='intersects') logging.info(f"Clipped data contains {len(gdf_clipped)} points.\n") except Exception as e: logging.error(f"Failed to clip data to Masovian boundary: {e}") sys.exit(1) # Step 8: Visualize and Plot try: logging.info("Visualizing radar data...") fig, ax = plt.subplots(figsize=(12, 12)) masovian_boundary.plot(ax=ax, edgecolor='black', facecolor='none', linewidth=1) scatter = gdf_clipped.plot( ax=ax, column='dBZ', cmap='jet', markersize=1, alpha=0.6, legend=True, legend_kwds={'label': "dBZ Value"} ) ax.set_title("Processed Radar dBZ Data within Masovian Region", fontsize=16) ax.set_xlabel("Easting (meters)", fontsize=12) ax.set_ylabel("Northing (meters)", fontsize=12) ax.set_aspect('equal') ax.set_xticks([]) ax.set_yticks([]) plt.savefig(OUTPUT_PLOT_PATH, dpi=300, bbox_inches='tight') plt.close() logging.info(f"Radar data visualization saved to {OUTPUT_PLOT_PATH}.\n") except Exception as e: logging.error(f"Failed to visualize radar data: {e}") sys.exit(1) logging.info("Radar data processing and visualization completed successfully.") if __name__ == "__main__": main()Explanation of Changes
-
Parsing the .vol File:
- Adjusted the
parse_vol_filefunction to read the radar data from the text file format. - Extracted elevation angles and dBZ values by parsing the lines of the file.
- Added error handling to exit if no dBZ data is found.
- Adjusted the
-
Creating Polar Grid:
- Created a function
create_polar_gridto generate polar coordinates based on elevations, range, and azimuths. - Used a fixed range of 125 km, assuming 0.5 km range bins (you can adjust this as needed).
- Created a function
-
Converting to Cartesian Coordinates:
- Adjusted the
polar_to_cartesianfunction to use the polar grid created. - Used the appropriate projection for converting polar coordinates centered at the radar location.
- Adjusted the
-
Creating the GeoDataFrame:
- Adjusted the
create_geodataframefunction to match the new data structures. - Flattened the arrays and ensured that the dBZ values correspond to the coordinates.
- Adjusted the
-
Visualization:
- Fixed the issue with the duplicated color bar by removing redundant color bar creation.
- Adjusted the plotting code to better represent the radar data extent.
Notes
-
Data Format Understanding:
Since the
dBZ.volfile is in a custom text format, the parsing code is based on assumptions about the data structure. Adjust the parsing logic according to the actual structure of your file. -
Azimuths and Elevations:
Ensure that the azimuth and elevation arrays correctly represent your data. The code assumes azimuths from 0 to 359 degrees with a 1-degree resolution.
-
Range Calculation:
You may need to adjust the range calculation based on the actual range bins in your data.
-
Masovian Boundary Clipping:
The clipping function now uses
gpd.sjointo spatially join the radar data with the Masovian boundary. Ensure that the Masovian boundary GeoJSON is correctly formatted and loaded.
Regarding Your Specific Issues:
-
Duplicated Color Bar:
The issue was likely due to multiple color bar creation in your original visualization code. In the updated code, we ensure that the color bar is only created once.
-
Extent of Radar Data:
By correctly computing the range and azimuths, and plotting the radar data over 125 km, the visualization should now reflect the expected extent.
-
Radar Data Representation:
By properly parsing the dBZ values from the file and matching them to the correct coordinates, the radar data plotted should better represent the actual data.
Additional Recommendations:
-
Verify Data Alignment:
Ensure that the number of dBZ values matches the number of coordinate points generated. If there is a mismatch, check the parsing logic and the data structure.
-
Adjust Resolutions:
You may need to adjust the azimuth and range resolutions to match your data's sampling.
-
Data Integrity Checks:
Implement additional checks to handle missing or invalid data in your radar file.
By adjusting the code to match the actual format of your
dBZ.volfile, you should be able to resolve the issues and generate the expected visualization of the radar data. -