import matplotlib.pyplot as plt import rasterio from rasterio.transform import…
- User
import matplotlib.pyplot as plt import rasterio from rasterio.transform import from_origin from rasterio.warp import calculate_default_transform, reproject, Resampling import geopandas as gpd from shapely.geometry import Point, Polygon from pyproj import Proj, Transformer import struct from tqdm import tqdm import os from lxml import etree import io import sys import logging import numpy as np # Added missing import import re import zlib # For data decompression
Configure logging
logging.basicConfig( level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s', handlers=[ logging.StreamHandler(sys.stdout), logging.FileHandler('radar_data_processing.log', mode='w') ] )
Define constants and paths
RADAR_DATA_PATH = 'input/radar/dBZ.vol' # Path to the radar data file MASOVIAN_BOUNDARY_PATH = 'input/radar/masovian_boundary.geojson' # Path to Masovian boundary GeoJSON or Shapefile OUTPUT_PLOT_PATH = 'output/processed_radar_data.png' # Path to save the output plot
Radar specifications based on provided XML
AZIMUTH_RESOLUTION = 1.0 # Degrees between each azimuth RANGE_RESOLUTION = 0.5 # As per 0.5 km
Radar location from
RADAR_LATITUDE = 52.405250 # From 52.405250 RADAR_LONGITUDE = 20.961110 # From 20.961110 RADAR_ALTITUDE = 122.3 # From 122.300000 meters
Coordinate Systems
SOURCE_CRS = 'EPSG:4326' # Geographic coordinates TARGET_CRS = 'EPSG:2180' # Polish coordinate system
def decompress_qt(blob_data): """ Decompresses data compressed with Qt's qCompress() function.
Parameters: blob_data: Compressed data including the 4-byte header. Returns: Decompressed data as bytes. """ if len(blob_data) < 4: raise ValueError("Data too short to contain Qt compressed header.") uncompressed_size = struct.unpack('>I', blob_data[:4])[0] compressed_data = blob_data[4:] decompressed_data = zlib.decompress(compressed_data) if len(decompressed_data) != uncompressed_size: logging.warning(f"Decompressed data size {len(decompressed_data)} does not match expected size {uncompressed_size}.") return decompressed_datadef parse_vol_file(file_path): """ Parses the .vol file to extract metadata about each radar slice.
Parameters: file_path: Path to the .vol file. Returns: slices_info: List of dictionaries containing slice metadata. sensor_info: Dictionary containing radar sensor information. blobs_data: Dictionary mapping blobids to their binary data. """ logging.info("Parsing the .vol file...") slices_info = [] sensor_info = {} blobs_data = {} with open(file_path, 'rb') as f: content = f.read() # Identify the end of the XML content end_xml_marker_pattern = re.compile(rb'<!--\s*END XML\s*-->', re.IGNORECASE) match = end_xml_marker_pattern.search(content) if not match: logging.error("<!-- END XML --> marker not found in the file.") sys.exit(1) end_xml_index = match.start() xml_content_bytes = content[:end_xml_index] # Exclude the END XML marker remaining_content = content[match.end():] # Parse XML try: root = etree.fromstring(xml_content_bytes) except etree.XMLSyntaxError as e: logging.error(f"Failed to parse XML content: {e}") sys.exit(1) # Extract sensor info sensor = root.find('.//sensorinfo') if sensor is not None: sensor_info['id'] = sensor.findtext('id') sensor_info['type'] = sensor.findtext('type') sensor_info['name'] = sensor.findtext('name') try: sensor_info['lon'] = float(sensor.findtext('lon')) sensor_info['lat'] = float(sensor.findtext('lat')) sensor_info['alt'] = float(sensor.findtext('alt')) sensor_info['wavelen'] = float(sensor.findtext('wavelen')) sensor_info['beamwidth'] = float(sensor.findtext('beamwidth')) except (TypeError, ValueError) as e: logging.error(f"Invalid sensor information: {e}") sys.exit(1) else: logging.error("sensorinfo not found in the XML.") sys.exit(1) # Extract slice information slices = root.findall('.//slice') for slice_elem in slices: slice_info = {} slice_info['refid'] = slice_elem.get('refid') # Find rawdata element for dBZ rawdata = slice_elem.find('.//rawdata[@type="dBZ"]') if rawdata is not None: try: slice_info['rays'] = int(rawdata.get('rays')) slice_info['bins'] = int(rawdata.get('bins')) slice_info['type'] = rawdata.get('type') slice_info['max'] = float(rawdata.get('max')) slice_info['min'] = float(rawdata.get('min')) slice_info['blobid'] = rawdata.get('blobid') slice_info['depth'] = int(rawdata.get('depth')) slices_info.append(slice_info) logging.info(f"Slice refid {slice_info['refid']}: rays {slice_info['rays']}, bins {slice_info['bins']}, blobid {slice_info['blobid']}.") except (TypeError, ValueError) as e: logging.warning(f"Invalid slice information in refid {slice_info.get('refid')}: {e}. Skipping this slice.") else: logging.warning(f"No dBZ rawdata found for slice refid {slice_elem.get('refid')}. Skipping this slice.") logging.info(f"Total slices with dBZ data: {len(slices_info)}.\n") # Now parse the BLOBs and collect the data blob_header_pattern = re.compile(rb'<BLOB\s+blobid="(\d+)"\s+size="(\d+)"\s+compression="([^"]*)"\s*>', re.IGNORECASE) offset = 0 # Offset within remaining_content while offset < len(remaining_content): match = blob_header_pattern.match(remaining_content, offset) if not match: # Skip any whitespace or non-matching content next_lt = remaining_content.find(b'<', offset + 1) if next_lt == -1: break offset = next_lt continue blobid = match.group(1).decode() size = int(match.group(2)) compression = match.group(3).decode() offset = match.end() # Move past the BLOB header # Read the blob data blob_data = remaining_content[offset:offset + size] offset += size # Move to the end of the blob data # Handle compression if necessary if compression == 'qt': # Decompress using zlib after removing 4-byte header try: blob_data = decompress_qt(blob_data) except Exception as e: logging.error(f"Failed to decompress blobid {blobid}: {e}") continue elif compression == 'none': pass # No decompression needed else: logging.warning(f"Unsupported compression '{compression}' for blobid {blobid}.") continue # Skip unsupported compression blobs_data[blobid] = blob_data logging.info(f"Read blobid {blobid} with size {len(blob_data)} bytes.") return slices_info, sensor_info, blobs_datadef extract_dBZ_data(slices_info, blobs_data): """ Extracts dBZ data from the blobs_data based on slice metadata.
Parameters: slices_info: List of dictionaries containing slice metadata. blobs_data: Dictionary mapping blobids to their binary data. Returns: dBZ_combined: 2D numpy array combining all slices (azimuths x range_bins). """ logging.info("Extracting dBZ data from blobs_data...") dBZ_list = [] for idx, slice_info in enumerate(slices_info): rays = slice_info['rays'] bins = slice_info['bins'] blobid = slice_info['blobid'] depth = slice_info.get('depth', 16) try: blob_data = blobs_data[blobid] except KeyError: logging.error(f"Blob data for blobid {blobid} not found. Skipping this slice.") continue # Determine data type based on depth if depth == 8: dtype = np.uint8 byte_size = 1 elif depth == 16: dtype = np.uint16 byte_size = 2 elif depth == 32: dtype = np.float32 byte_size = 4 else: logging.warning(f"Unsupported depth {depth} for slice refid {slice_info['refid']}. Skipping this slice.") continue expected_bytes = rays * bins * byte_size # Check if blob_data size matches expected size if len(blob_data) != expected_bytes: logging.warning(f"Blob data size {len(blob_data)} does not match expected size {expected_bytes} for slice refid {slice_info['refid']}. Adjusting bins.") # Adjust bins accordingly valid_bins = len(blob_data) // (rays * byte_size) if valid_bins <= 0: logging.error(f"Slice {idx + 1} refid {slice_info['refid']}: No valid data after adjustment. Skipping this slice.") continue slice_info['bins'] = valid_bins bins = valid_bins expected_bytes = rays * bins * byte_size try: # Convert binary data to numpy array dBZ = np.frombuffer(blob_data[:expected_bytes], dtype=dtype).reshape((rays, bins)) # Scale data to actual dBZ values scale = (slice_info['max'] - slice_info['min']) / (2 ** depth - 2) offset = slice_info['min'] dBZ = dBZ * scale + offset dBZ_list.append(dBZ) logging.info(f"Slice {idx + 1} refid {slice_info['refid']}: Extracted dBZ data with shape {dBZ.shape}.") except Exception as e: logging.error(f"Slice {idx + 1} refid {slice_info['refid']}: Failed to parse dBZ data. Error: {e}. Skipping this slice.") continue if not dBZ_list: logging.error("No dBZ data extracted. Exiting.") sys.exit(1) # Ensure all slices have the same number of bins by padding with np.nan max_bins = max(dBZ.shape[1] for dBZ in dBZ_list) logging.info(f"Maximum number of bins across slices: {max_bins}") padded_dBZ_list = [] for idx, dBZ in enumerate(dBZ_list): current_bins = dBZ.shape[1] if current_bins < max_bins: padding = ((0, 0), (0, max_bins - current_bins)) dBZ_padded = np.pad(dBZ, padding, mode='constant', constant_values=np.nan) logging.info(f"Padded slice {idx + 1} from {current_bins} to {max_bins} bins.") else: dBZ_padded = dBZ padded_dBZ_list.append(dBZ_padded) # Stack all slices vertically try: dBZ_combined = np.vstack(padded_dBZ_list) logging.info(f"Combined dBZ data shape after padding and stacking: {dBZ_combined.shape}\n") except Exception as e: logging.error(f"Failed to stack dBZ data: {e}") sys.exit(1) return dBZ_combineddef polar_to_cartesian(dBZ, az_res, rng_res, max_range=None): """ Converts polar radar data to Cartesian coordinates.
Parameters: dBZ: 2D numpy array of dBZ values (azimuths x ranges). az_res: Azimuth resolution in degrees. rng_res: Range resolution in km. max_range: Maximum radar range in km. Returns: xs: 1D numpy array of x coordinates in meters. ys: 1D numpy array of y coordinates in meters. dBZ_1D: 1D numpy array of dBZ values. """ logging.info("Converting polar coordinates to Cartesian coordinates...") num_az, num_rng = dBZ.shape # Ensure that azimuths cover 0 to 360 degrees azimuths = np.deg2rad(np.linspace(0, 360, num=num_az, endpoint=False)) # Calculate ranges ranges = np.linspace(rng_res / 2, num_rng * rng_res - rng_res / 2, num=num_rng) * 1000 # Convert km to meters if max_range is not None: expected_bins = int(max_range / rng_res) if num_rng < expected_bins: logging.warning(f"Number of range bins ({num_rng}) is less than expected ({expected_bins}). Adjusting max range.") max_range = num_rng * rng_res elif num_rng > expected_bins: logging.warning(f"Number of range bins ({num_rng}) is greater than expected ({expected_bins}). Truncating.") dBZ = dBZ[:, :expected_bins] num_rng = expected_bins ranges = ranges[:expected_bins] # Create meshgrid az, r = np.meshgrid(azimuths, ranges, indexing='ij') # Convert to Cartesian coordinates x = r * np.sin(az) y = r * np.cos(az) # Flatten the arrays xs = x.flatten() ys = y.flatten() dBZ_1D = dBZ.flatten() logging.info("Coordinate conversion completed.\n") return xs, ys, dBZ_1Ddef create_geodataframe(xs, ys, dBZ): """ Creates a GeoDataFrame from Cartesian coordinates and dBZ values.
Parameters: xs: 1D numpy array of x coordinates in meters. ys: 1D numpy array of y coordinates in meters. dBZ: 1D numpy array of dBZ values. Returns: gdf: GeoDataFrame with geometry in geographic coordinates. """ logging.info("Creating GeoDataFrame...") # Define local projection (using Azimuthal Equidistant projection centered at radar location) proj_local = Proj(proj='aeqd', lat_0=RADAR_LATITUDE, lon_0=RADAR_LONGITUDE, ellps='WGS84', units='m') transformer = Transformer.from_proj(proj_local, 'epsg:4326', always_xy=True) # Transform to geographic coordinates lon, lat = transformer.transform(xs, ys) # Create GeoDataFrame geometry = [Point(xy) for xy in zip(lon, lat)] gdf = gpd.GeoDataFrame({'dBZ': dBZ}, geometry=geometry, crs='EPSG:4326') logging.info("GeoDataFrame creation completed.\n") return gdfdef reproject_gdf(gdf, target_crs='EPSG:2180'): """ Reprojects the GeoDataFrame to the target CRS.
Parameters: gdf: Input GeoDataFrame. target_crs: Target Coordinate Reference System. Returns: gdf_reprojected: Reprojected GeoDataFrame. """ logging.info(f"Reprojecting GeoDataFrame to {target_crs}...") try: gdf_reprojected = gdf.to_crs(target_crs) logging.info("Reprojection completed.\n") except Exception as e: logging.error(f"Failed to reproject GeoDataFrame: {e}") sys.exit(1) return gdf_reprojecteddef clip_to_masovian(gdf, masovian_boundary): """ Clips the GeoDataFrame to the Masovian boundary.
Parameters: gdf: Input GeoDataFrame. masovian_boundary: GeoDataFrame or Polygon of Masovian boundary. Returns: gdf_clipped: Clipped GeoDataFrame. """ logging.info("Clipping radar data to Masovian boundary...") try: # Ensure geometries are valid masovian_boundary = masovian_boundary.unary_union gdf_clipped = gdf[gdf.geometry.within(masovian_boundary)] logging.info(f"Clipped data contains {len(gdf_clipped)} points.\n") except Exception as e: logging.error(f"Failed to clip data to Masovian boundary: {e}") sys.exit(1) return gdf_clippeddef visualize_radar_data(gdf, masovian_boundary, output_path): """ Visualizes the radar data.
Parameters: gdf: GeoDataFrame containing clipped radar data. masovian_boundary: GeoDataFrame or Polygon of Masovian boundary. output_path: Path to save the plot. Returns: None """ logging.info("Visualizing radar data...") try: fig, ax = plt.subplots(figsize=(12, 12)) # Plot Masovian boundary masovian_boundary.boundary.plot(ax=ax, edgecolor='black', linewidth=1) # Plot radar dBZ values using scatter plot # Filter out invalid dBZ values (e.g., NaN or extreme values) valid_gdf = gdf[(gdf['dBZ'] >= -30) & (gdf['dBZ'] <= 95.5)] scatter = ax.scatter( valid_gdf.geometry.x, valid_gdf.geometry.y, c=valid_gdf['dBZ'], cmap='jet', s=5, alpha=0.6 ) ax.set_title("Processed Radar dBZ Data within Masovian Region", fontsize=16) ax.set_xlabel("Easting (meters)", fontsize=12) ax.set_ylabel("Northing (meters)", fontsize=12) ax.set_aspect('equal') # Remove axis ticks for cleaner look ax.set_xticks([]) ax.set_yticks([]) # Add colorbar manually cbar = fig.colorbar(scatter, ax=ax, fraction=0.036, pad=0.04) cbar.set_label('dBZ', fontsize=12) # Save the plot plt.savefig(output_path, dpi=300, bbox_inches='tight') plt.close() logging.info(f"Radar data visualization saved to {output_path}.\n") except Exception as e: logging.error(f"Failed to visualize radar data: {e}") sys.exit(1)def main(): # Step 1: Parse the .vol file if not os.path.exists(RADAR_DATA_PATH): logging.error(f"Radar data file not found at {RADAR_DATA_PATH}. Please check the path.") sys.exit(1)
slices_info, sensor_info, blobs_data = parse_vol_file(RADAR_DATA_PATH) # Step 2: Extract dBZ data from blobs_data dBZ_combined = extract_dBZ_data(slices_info, blobs_data) # Step 3: Convert polar to Cartesian coordinates xs, ys, dBZ_1D = polar_to_cartesian( dBZ_combined, AZIMUTH_RESOLUTION, RANGE_RESOLUTION ) # Step 4: Create GeoDataFrame gdf_geo = create_geodataframe(xs, ys, dBZ_1D) # Step 5: Reproject to EPSG:2180 gdf_proj = reproject_gdf(gdf_geo, TARGET_CRS) # Step 6: Load Masovian Boundary if not os.path.exists(MASOVIAN_BOUNDARY_PATH): logging.error(f"Masovian boundary file not found at {MASOVIAN_BOUNDARY_PATH}. Please check the path.") sys.exit(1) try: masovian_boundary = gpd.read_file(MASOVIAN_BOUNDARY_PATH) logging.info("Loaded Masovian boundary successfully.\n") except Exception as e: logging.error(f"Failed to load Masovian boundary: {e}") sys.exit(1) # Ensure Masovian boundary is in the same CRS as target CRS if masovian_boundary.crs != TARGET_CRS: try: masovian_boundary = masovian_boundary.to_crs(TARGET_CRS) logging.info(f"Reprojected Masovian boundary to {TARGET_CRS}.\n") except Exception as e: logging.error(f"Failed to reproject Masovian boundary: {e}") sys.exit(1) # Step 7: Clip Radar Data to Masovian Boundary gdf_clipped = clip_to_masovian(gdf_proj, masovian_boundary) # Step 8: Visualize and Plot visualize_radar_data(gdf_clipped, masovian_boundary, OUTPUT_PLOT_PATH) logging.info("Radar data processing and visualization completed successfully.")if name == "main": main()
Fix the error: 2024-10-05 23:06:56,100 - INFO - Parsing the .vol file... 2024-10-05 23:06:56,100 - INFO - Slice refid 0: rays 360, bins 250, blobid 2. 2024-10-05 23:06:56,100 - INFO - Slice refid 1: rays 360, bins 250, blobid 5. 2024-10-05 23:06:56,100 - INFO - Slice refid 2: rays 360, bins 250, blobid 8. 2024-10-05 23:06:56,100 - INFO - Slice refid 3: rays 360, bins 250, blobid 11. 2024-10-05 23:06:56,100 - INFO - Slice refid 4: rays 360, bins 340, blobid 14. 2024-10-05 23:06:56,100 - INFO - Slice refid 5: rays 360, bins 250, blobid 17. 2024-10-05 23:06:56,100 - INFO - Slice refid 6: rays 360, bins 250, blobid 20. 2024-10-05 23:06:56,100 - INFO - Slice refid 7: rays 360, bins 250, blobid 23. 2024-10-05 23:06:56,100 - INFO - Total slices with dBZ data: 8.
2024-10-05 23:06:56,100 - ERROR - Failed to decompress blobid 0: Error -3 while decompressing data: incorrect header check 2024-10-05 23:06:56,100 - ERROR - Failed to decompress blobid 1: Error -3 while decompressing data: incorrect header check 2024-10-05 23:06:56,100 - ERROR - Failed to decompress blobid 2: Error -3 while decompressing data: incorrect header check 2024-10-05 23:06:56,100 - ERROR - Failed to decompress blobid 3: Error -3 while decompressing data: incorrect header check 2024-10-05 23:06:56,100 - ERROR - Failed to decompress blobid 4: Error -3 while decompressing data: incorrect header check 2024-10-05 23:06:56,100 - ERROR - Failed to decompress blobid 5: Error -3 while decompressing data: incorrect header check 2024-10-05 23:06:56,100 - ERROR - Failed to decompress blobid 6: Error -3 while decompressing data: incorrect header check 2024-10-05 23:06:56,100 - ERROR - Failed to decompress blobid 7: Error -3 while decompressing data: incorrect header check 2024-10-05 23:06:56,100 - ERROR - Failed to decompress blobid 8: Error -3 while decompressing data: incorrect header check 2024-10-05 23:06:56,100 - ERROR - Failed to decompress blobid 9: Error -3 while decompressing data: incorrect header check 2024-10-05 23:06:56,100 - ERROR - Failed to decompress blobid 10: Error -3 while decompressing data: incorrect header check 2024-10-05 23:06:56,100 - ERROR - Failed to decompress blobid 11: Error -3 while decompressing data: incorrect header check 2024-10-05 23:06:56,100 - ERROR - Failed to decompress blobid 12: Error -3 while decompressing data: incorrect header check 2024-10-05 23:06:56,100 - ERROR - Failed to decompress blobid 13: Error -3 while decompressing data: incorrect header check 2024-10-05 23:06:56,116 - ERROR - Failed to decompress blobid 14: Error -3 while decompressing data: incorrect header check 2024-10-05 23:06:56,116 - ERROR - Failed to decompress blobid 15: Error -3 while decompressing data: incorrect header check 2024-10-05 23:06:56,116 - ERROR - Failed to decompress blobid 16: Error -3 while decompressing data: incorrect header check 2024-10-05 23:06:56,116 - ERROR - Failed to decompress blobid 17: Error -3 while decompressing data: incorrect header check 2024-10-05 23:06:56,116 - ERROR - Failed to decompress blobid 18: Error -3 while decompressing data: incorrect header check 2024-10-05 23:06:56,116 - ERROR - Failed to decompress blobid 19: Error -3 while decompressing data: incorrect header check 2024-10-05 23:06:56,117 - ERROR - Failed to decompress blobid 20: Error -3 while decompressing data: incorrect header check 2024-10-05 23:06:56,117 - ERROR - Failed to decompress blobid 21: Error -3 while decompressing data: incorrect header check 2024-10-05 23:06:56,117 - ERROR - Failed to decompress blobid 22: Error -3 while decompressing data: incorrect header check 2024-10-05 23:06:56,118 - ERROR - Failed to decompress blobid 23: Error -3 while decompressing data: incorrect header check 2024-10-05 23:06:56,118 - INFO - Extracting dBZ data from blobs_data... 2024-10-05 23:06:56,118 - ERROR - Blob data for blobid 2 not found. Skipping this slice. 2024-10-05 23:06:56,118 - ERROR - Blob data for blobid 5 not found. Skipping this slice. 2024-10-05 23:06:56,119 - ERROR - Blob data for blobid 8 not found. Skipping this slice. 2024-10-05 23:06:56,119 - ERROR - Blob data for blobid 11 not found. Skipping this slice. 2024-10-05 23:06:56,119 - ERROR - Blob data for blobid 14 not found. Skipping this slice. 2024-10-05 23:06:56,119 - ERROR - Blob data for blobid 17 not found. Skipping this slice. 2024-10-05 23:06:56,119 - ERROR - Blob data for blobid 20 not found. Skipping this slice. 2024-10-05 23:06:56,120 - ERROR - Blob data for blobid 23 not found. Skipping this slice. 2024-10-05 23:06:56,120 - ERROR - No dBZ data extracted. Exiting.
dBZ.vol: SI 0 1 249 2024-10-05T18:46:32 2024-10-05T18:49:14 2024-10-05T18:46:30 2024-10-05T18:51:30 Off 2 3 Off Off <phase_coding>2</phase_coding> <phase_pattern>0</phase_pattern> 250 8 Optimised 0.5 40 0 Off <gdrx_dp_proc_mode>GdrxDpModeHV_HV</gdrx_dp_proc_mode> 550.00 None 0.00 0.00 2.05 FreePRF <start_range>0</start_range> 0.5 1 Off 24 1 23 <dynpw_matchedrangeresolution>None</dynpw_matchedrangeresolution> <pw_index>0</pw_index> <cf_doppler>Off</cf_doppler> On FFT <cf_spatial>On</cf_spatial> Off <gdrx_clutter_flag_filter>Off</gdrx_clutter_flag_filter> <cf_dectree>Off</cf_dectree> <cf_gip>Off</cf_gip> NoAlgType 6 <cf_gip_width>0.5000</cf_gip_width> <cf_gip_mode_const>On</cf_gip_mode_const> <gdrx_cluttermap>GdrxCmOff</gdrx_cluttermap> <int_filter_mode>1</int_filter_mode> <int_filter_corr_mode>Interpolate</int_filter_corr_mode> <int_filter_hconf>18</int_filter_hconf> <int_filter_vconf>18</int_filter_vconf> <int_filter_phaseint>On</int_filter_phaseint> AngleStep false 30 0.3 0.5 2 SdfDPrfModeAda 2 On SdfDPrfCorrMedian SdfCfDepthDefault <fft_weight>GdrxWfHamming</fft_weight> <spatial_modus>SdfSpatialFilterAvg</spatial_modus> <spatial_min>3</spatial_min> <spatial_max>5</spatial_max> <gdrx_tripnotchwidth>0.5</gdrx_tripnotchwidth> <gdrx_tripccorthres>20</gdrx_tripccorthres> <gdrx_tripsnrthres>3</gdrx_tripsnrthres> <gdrx_tripsqithres>0.4</gdrx_tripsqithres> <gdrx_tripdftwin>GdrxWfRectangular</gdrx_tripdftwin> CCOR,CCORv,dBZ,dBuZ,dBZv,dBuZv,FLAGS,uKDP,KDP,uPhiDP,PhiDP,RhoHV,SNR,SNRu,SNRv,SQI,SQIv,V,W,ZDR CCOR,dBZ,FLAGS,KDP,PhiDP,RhoHV,SNR,SQI,V,W,ZDR <datatypes_format>dBZ=0,dBZv=0,dBuZ=0,dBuZv=0,V=0,Vv=0,Vu=0,Vvu=0,W=0,Wv=0,Wu=0,Wvu=0,ZDR=0,ZDRu=0,uPhiDP=1,uPhiDPu=1,PhiDP=1,KDP=0,uKDP=0,uKDPu=0,RhoHV=0,RhoHVu=0,CCOR=0,CCORv=0,SNR=1,SNRv=0,SNRu=0,SNRvu=0,SQI=0,SQIu=0,SQIv=0,SQIvu=0,FLAGS=-1</datatypes_format> 23.800000,15.000000,9.100000,5.400000,3.500000,2.500000,1.500000,0.500000,3.500000,2.500000,1.500000,0.500000 0.50 <dynpw_constantdc_enabled>Off</dynpw_constantdc_enabled> <dynpw_constantdc>0.00</dynpw_constantdc> Off 0.5 18 29 DATA <cf_gip_width>0.4</cf_gip_width> <int_filter_mode>3</int_filter_mode> <int_filter_hconf>10</int_filter_hconf> <int_filter_vconf>10</int_filter_vconf> 0 None dBZ:622602,dBZv:622602,dBuZ:1,dBuZv:1,V:524294,Vv:524294,Vu:1,Vvu:1,W:524294,Wv:524294,Wu:1,Wvu:1,ZDR:622602,ZDRu:1,uPhiDP:2,uPhiDPu:1,PhiDP:524290,KDP:524290,uKDP:2,uKDPu:1,RhoHV:622594,RhoHVu:1 2 3 125 <start_range>0</start_range> 0.5 4 Off <cf_doppler>Off</cf_doppler> <fft_weight>GdrxWfHamming</fft_weight> <spatial_modus>SdfSpatialFilterAvg</spatial_modus> <spatial_min>3</spatial_min> <spatial_max>5</spatial_max> 0.0 0.0 1200 4/3 SdfDPrfCorrMedian On 2 SdfDPrfModeAda 900.00 Off 4 SdfCfDepthDefault Off <cf_spatial>Off</cf_spatial> Off 0 AngleStep 1 0 On 30 0.1 0.1 2 On <phase_coding>2</phase_coding> <gdrx_cluttermap>GdrxCmOff</gdrx_cluttermap> <gdrx_tripnotchwidth>0.5</gdrx_tripnotchwidth> <gdrx_tripccorthres>20</gdrx_tripccorthres> <gdrx_tripsnrthres>3</gdrx_tripsnrthres> <gdrx_tripsqithres>0.4</gdrx_tripsqithres> <gdrx_tripdftwin>GdrxWfRectangular</gdrx_tripdftwin> <gdrx_clutter_flag_filter>Off</gdrx_clutter_flag_filter> <cf_gip>On</cf_gip> <cf_gip_mode_const>On</cf_gip_mode_const> <cf_gip_max_width>0.5000</cf_gip_max_width> On <int_filter_phaseint>On</int_filter_phaseint> <int_filter_corr_mode>Interpolate</int_filter_corr_mode> <cf_dectree>Off</cf_dectree> On Off 0.00 SdfStaticLookupFilter 0.95 <dynpw_constantdc_enabled>On</dynpw_constantdc_enabled> <dynpw_constantdc>80</dynpw_constantdc> <dynpw_matchedrangeresolution>None</dynpw_matchedrangeresolution> FreePRF 3.00 8 12 28462 6139 <radarprefix_sp>262plc1_sq</radarprefix_sp> <radarprefix_rcp>262plc1_sq_lk</radarprefix_rcp> OK OK 5649.8447 OK 93.3 0 0.019 5.31 122.3 OK 45.38 20.7 217.77 45.37 21.08 3.51 0.0 210.86 0.66 0.95 0.00 0.2 0.93 3.32 0.0 217.77 0.66 0.99 OK OK 1100000 -0.15 <noise_power_dbz>-42.2651</noise_power_dbz> <noise_power_dbz_dpv>-42.3474</noise_power_dbz_dpv> 69.534 69.534 69.884 4 20 0.0 59.8483 1.5 7 6138 59.9227 2.5 6 6137 60.0078 3.5 5 6136 <noise_power_dbz_dpv>-42.3475</noise_power_dbz_dpv> 60.0972 5.4 170 882 661.50 1.29 16 <int_filter_hconf>6</int_filter_hconf> <int_filter_vconf>6</int_filter_vconf> 24 <cf_gip_width>0.3</cf_gip_width> 4 6135 800000 <noise_power_dbz>-44.9218</noise_power_dbz> <noise_power_dbz_dpv>-45.0041</noise_power_dbz_dpv> 68.206 68.206 68.556 60.2026 9.1 20 26 DATA,NOISE <cf_gip_width>0.3</cf_gip_width> 3 6134 60.2829 15.0 24 DATA,NOISE 22 <cf_gip_width>0.3</cf_gip_width> <int_filter_mode>2</int_filter_mode> <int_filter_hconf>16</int_filter_hconf> <int_filter_vconf>16</int_filter_vconf> 2 6133 60.111 23.8 <cf_gip_width>0.3</cf_gip_width> <int_filter_mode>2</int_filter_mode> <int_filter_hconf>16</int_filter_hconf> <int_filter_vconf>16</int_filter_vconf> 22 24 DATA,NOISE 1 6132 <noise_power_dbz_dpv>-42.3475</noise_power_dbz_dpv> 59.9191 20.961110 52.405250 122.300000 0.0531 0.95
- Assistant
The error you're encountering—
Error -3 while decompressing data: incorrect header check—indicates that the decompression process is failing due to an incorrect header in the compressed data. This typically happens when there's a mismatch between the expected and actual data formats or byte orders.Upon reviewing your code and the provided
.volfile, the primary issue lies in how the uncompressed size is being extracted from the compressed blob data. Specifically, thedecompress_qtfunction is using big-endian ('>I') byte order to unpack the first four bytes, whereas Qt'sqCompressfunction actually uses little-endian ('<I') byte order for the header.Detailed Explanation
-
Qt's
qCompressFunction:- Qt's
qCompressfunction compresses data and prepends a 4-byte header indicating the size of the uncompressed data. - This 4-byte header is in little-endian format.
- Qt's
-
Current Implementation:
- Your
decompress_qtfunction usesstruct.unpack('>I', blob_data[:4])[0]to interpret the header. - Here,
'>I'denotes a big-endian unsigned integer, which does not match Qt's little-endian format.
- Your
-
Impact of Mismatched Byte Order:
- Using the wrong byte order leads to incorrect interpretation of the uncompressed size.
- Consequently, the decompression algorithm (
zlib.decompress) receives incorrectly sized or malformed data, resulting in theincorrect header checkerror.
Solution
To fix this issue, you need to change the byte order in the
struct.unpackfunction from big-endian ('>I') to little-endian ('<I'). This ensures that the uncompressed size is correctly interpreted, allowingzlibto decompress the data properly.Updated Code
Here's the corrected version of your
decompress_qtfunction along with the necessary adjustments in theparse_vol_filefunction to ensure only relevant blobs are processed:import matplotlib.pyplot as plt import rasterio from rasterio.transform import from_origin from rasterio.warp import calculate_default_transform, reproject, Resampling import geopandas as gpd from shapely.geometry import Point, Polygon from pyproj import Proj, Transformer import struct from tqdm import tqdm import os from lxml import etree import io import sys import logging import numpy as np # Added missing import import re import zlib # For data decompression # Configure logging logging.basicConfig( level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s', handlers=[ logging.StreamHandler(sys.stdout), logging.FileHandler('radar_data_processing.log', mode='w') ] ) # Define constants and paths RADAR_DATA_PATH = 'input/radar/dBZ.vol' # Path to the radar data file MASOVIAN_BOUNDARY_PATH = 'input/radar/masovian_boundary.geojson' # Path to Masovian boundary GeoJSON or Shapefile OUTPUT_PLOT_PATH = 'output/processed_radar_data.png' # Path to save the output plot # Radar specifications based on provided XML AZIMUTH_RESOLUTION = 1.0 # Degrees between each azimuth RANGE_RESOLUTION = 0.5 # As per 0.5 km # Radar location from RADAR_LATITUDE = 52.405250 # From 52.405250 RADAR_LONGITUDE = 20.961110 # From 20.961110 RADAR_ALTITUDE = 122.3 # From 122.300000 meters # Coordinate Systems SOURCE_CRS = 'EPSG:4326' # Geographic coordinates TARGET_CRS = 'EPSG:2180' # Polish coordinate system def decompress_qt(blob_data): """ Decompresses data compressed with Qt's qCompress() function. Parameters: blob_data: Compressed data including the 4-byte header. Returns: Decompressed data as bytes. """ if len(blob_data) < 4: raise ValueError("Data too short to contain Qt compressed header.") # Change from big-endian to little-endian uncompressed_size = struct.unpack('<I', blob_data[:4])[0] compressed_data = blob_data[4:] try: decompressed_data = zlib.decompress(compressed_data) except zlib.error as e: logging.error(f"Zlib decompression error: {e}") raise if len(decompressed_data) != uncompressed_size: logging.warning(f"Decompressed data size {len(decompressed_data)} does not match expected size {uncompressed_size}.") return decompressed_data def parse_vol_file(file_path): """ Parses the .vol file to extract metadata about each radar slice. Parameters: file_path: Path to the .vol file. Returns: slices_info: List of dictionaries containing slice metadata. sensor_info: Dictionary containing radar sensor information. blobs_data: Dictionary mapping blobids to their binary data. """ logging.info("Parsing the .vol file...") slices_info = [] sensor_info = {} blobs_data = {} with open(file_path, 'rb') as f: content = f.read() # Identify the end of the XML content end_xml_marker_pattern = re.compile(rb'<!--\s*END XML\s*-->', re.IGNORECASE) match = end_xml_marker_pattern.search(content) if not match: logging.error("<!-- END XML --> marker not found in the file.") sys.exit(1) end_xml_index = match.start() xml_content_bytes = content[:end_xml_index] # Exclude the END XML marker remaining_content = content[match.end():] # Parse XML try: root = etree.fromstring(xml_content_bytes) except etree.XMLSyntaxError as e: logging.error(f"Failed to parse XML content: {e}") sys.exit(1) # Extract sensor info sensor = root.find('.//sensorinfo') if sensor is not None: sensor_info['id'] = sensor.findtext('id') sensor_info['type'] = sensor.findtext('type') sensor_info['name'] = sensor.findtext('name') try: sensor_info['lon'] = float(sensor.findtext('lon')) sensor_info['lat'] = float(sensor.findtext('lat')) sensor_info['alt'] = float(sensor.findtext('alt')) sensor_info['wavelen'] = float(sensor.findtext('wavelen')) sensor_info['beamwidth'] = float(sensor.findtext('beamwidth')) except (TypeError, ValueError) as e: logging.error(f"Invalid sensor information: {e}") sys.exit(1) else: logging.error("sensorinfo not found in the XML.") sys.exit(1) # Extract slice information slices = root.findall('.//slice') for slice_elem in slices: slice_info = {} slice_info['refid'] = slice_elem.get('refid') # Find rawdata element for dBZ rawdata = slice_elem.find('.//rawdata[@type="dBZ"]') if rawdata is not None: try: slice_info['rays'] = int(rawdata.get('rays')) slice_info['bins'] = int(rawdata.get('bins')) slice_info['type'] = rawdata.get('type') slice_info['max'] = float(rawdata.get('max')) slice_info['min'] = float(rawdata.get('min')) slice_info['blobid'] = rawdata.get('blobid') slice_info['depth'] = int(rawdata.get('depth')) slices_info.append(slice_info) logging.info(f"Slice refid {slice_info['refid']}: rays {slice_info['rays']}, bins {slice_info['bins']}, blobid {slice_info['blobid']}.") except (TypeError, ValueError) as e: logging.warning(f"Invalid slice information in refid {slice_info.get('refid')}: {e}. Skipping this slice.") else: logging.warning(f"No dBZ rawdata found for slice refid {slice_elem.get('refid')}. Skipping this slice.") logging.info(f"Total slices with dBZ data: {len(slices_info)}.\n") # Collect required blobids for dBZ data required_blobids = set(slice_info['blobid'] for slice_info in slices_info) # Now parse the BLOBs and collect the data blob_header_pattern = re.compile(rb'<BLOB\s+blobid="(\d+)"\s+size="(\d+)"\s+compression="([^"]*)"\s*>', re.IGNORECASE) offset = 0 # Offset within remaining_content while offset < len(remaining_content): match = blob_header_pattern.match(remaining_content, offset) if not match: # Skip any whitespace or non-matching content next_lt = remaining_content.find(b'<', offset + 1) if next_lt == -1: break offset = next_lt continue blobid = match.group(1).decode() size = int(match.group(2)) compression = match.group(3).decode() offset = match.end() # Move past the BLOB header # Read the blob data blob_data = remaining_content[offset:offset + size] offset += size # Move to the end of the blob data # Handle compression if necessary if blobid in required_blobids: if compression == 'qt': # Decompress using zlib after removing 4-byte header try: blob_data = decompress_qt(blob_data) blobs_data[blobid] = blob_data logging.info(f"Read and decompressed blobid {blobid} with size {len(blob_data)} bytes.") except Exception as e: logging.error(f"Failed to decompress blobid {blobid}: {e}") elif compression == 'none': blobs_data[blobid] = blob_data logging.info(f"Read blobid {blobid} with size {len(blob_data)} bytes (no compression).") else: logging.warning(f"Unsupported compression '{compression}' for blobid {blobid}.") else: # Skip blobids not needed for dBZ data logging.info(f"Skipping blobid {blobid} as it is not required for dBZ data.") return slices_info, sensor_info, blobs_data # ... [rest of your code remains unchanged] ... def main(): # Step 1: Parse the .vol file if not os.path.exists(RADAR_DATA_PATH): logging.error(f"Radar data file not found at {RADAR_DATA_PATH}. Please check the path.") sys.exit(1) slices_info, sensor_info, blobs_data = parse_vol_file(RADAR_DATA_PATH) # Step 2: Extract dBZ data from blobs_data dBZ_combined = extract_dBZ_data(slices_info, blobs_data) # Step 3: Convert polar to Cartesian coordinates xs, ys, dBZ_1D = polar_to_cartesian( dBZ_combined, AZIMUTH_RESOLUTION, RANGE_RESOLUTION ) # Step 4: Create GeoDataFrame gdf_geo = create_geodataframe(xs, ys, dBZ_1D) # Step 5: Reproject to EPSG:2180 gdf_proj = reproject_gdf(gdf_geo, TARGET_CRS) # Step 6: Load Masovian Boundary if not os.path.exists(MASOVIAN_BOUNDARY_PATH): logging.error(f"Masovian boundary file not found at {MASOVIAN_BOUNDARY_PATH}. Please check the path.") sys.exit(1) try: masovian_boundary = gpd.read_file(MASOVIAN_BOUNDARY_PATH) logging.info("Loaded Masovian boundary successfully.\n") except Exception as e: logging.error(f"Failed to load Masovian boundary: {e}") sys.exit(1) # Ensure Masovian boundary is in the same CRS as target CRS if masovian_boundary.crs != TARGET_CRS: try: masovian_boundary = masovian_boundary.to_crs(TARGET_CRS) logging.info(f"Reprojected Masovian boundary to {TARGET_CRS}.\n") except Exception as e: logging.error(f"Failed to reproject Masovian boundary: {e}") sys.exit(1) # Step 7: Clip Radar Data to Masovian Boundary gdf_clipped = clip_to_masovian(gdf_proj, masovian_boundary) # Step 8: Visualize and Plot visualize_radar_data(gdf_clipped, masovian_boundary, OUTPUT_PLOT_PATH) logging.info("Radar data processing and visualization completed successfully.") if __name__ == "__main__": main()Key Changes Explained
-
Byte Order Correction:
- Before:
uncompressed_size = struct.unpack('>I', blob_data[:4])[0] - After:
uncompressed_size = struct.unpack('<I', blob_data[:4])[0] - Explanation: Changed from big-endian (
'>I') to little-endian ('<I') to correctly interpret the uncompressed size.
- Before:
-
Selective Decompression:
- Problem: The original code attempted to decompress all blobs, including those not related to
dBZdata (e.g.,rayinfoblobs), leading to errors. - Solution: Modify the
parse_vol_filefunction to only decompress blobs that are relevant todBZdata. - Implementation:
- Extract the set of
blobids required fordBZfromslices_info. - Only attempt to decompress blobs that are in this set.
- Skip decompressing irrelevant blobs, thereby avoiding unnecessary errors.
- Extract the set of
- Problem: The original code attempted to decompress all blobs, including those not related to
-
Enhanced Error Handling:
- Added a try-except block around the
zlib.decompresscall to catch and log any decompression errors more gracefully.
- Added a try-except block around the
Additional Recommendations
-
Verify Compression Types:
- Ensure that all
dBZblobs indeed usecompression="qt". If some use a different compression method, you'll need to handle them accordingly.
- Ensure that all
-
Validate Blob Sizes:
- After decompression, verify that the blob sizes match the expected dimensions (
rays x bins). If there's a mismatch, it might indicate data corruption or an incorrect decompression process.
- After decompression, verify that the blob sizes match the expected dimensions (
-
Logging Enhancements:
- Consider adding more detailed logs or intermediate checks to trace the decompression and data extraction steps more effectively.
-
Testing:
- After making these changes, test the script with a subset of data to ensure that decompression and data extraction are working as expected before processing the entire dataset.
By implementing these changes, your script should correctly decompress the
dBZblobs and proceed with data extraction and visualization without encountering the previous decompression errors. -