Skip to content
TheChatArchiveThe Chat Archive
Geminigemini-2.0-flash-001· GoogleHugging Face LMArena Human Preference 140K

Вот код: [code] Нужно его переделать так, что бы в результате получались три датафрейма:…

From a Chatbot Arena battleThe same person sent this prompt to two anonymous models at once. Compare it with Claudeclaude-3-5-sonnet-20241022’s answer.
  1. User

    Вот код:

    #%% md
    # Ноутбук для улучшения связки BAYUT и PF
    #%%
    import os
    import sys
    from pathlib import Path
    
    from modules.google_drive_manager import GoogleDriveManager
    
    this_file_dir = Path.cwd()
    sys.path.append(str(this_file_dir.parent))
    
    bayut_buildings_path = os.path.join(this_file_dir.parent, "cloud_data", "bayut_buildings.parquet")
    pf_buildings_path = os.path.join(this_file_dir.parent, "cloud_data", "pf_buildings.parquet")
    mapping_path = os.path.join(this_file_dir.parent, "cloud_data", "building_names_with_geo.pkl")
    connection_ids_path = os.path.join(this_file_dir.parent, "cloud_data", "connection_IDS.pkl")
    
    building_desc_info_path = os.path.join(
        this_file_dir.parent, "cloud_data", "building_description_info.json"
    )
    
    # LOADING DATA FROM DRIVE
    drive_folder_id = "19i06Z_3xx1FDmIXWpijG_Jzm37eteySd"
    manager = GoogleDriveManager(
        google_service_account_file=os.path.join(
            this_file_dir.parent, "cloud_auth_file", "dubai-estate-7831cb4f621b.json"
        ),
        download_info_file_path=os.path.join(this_file_dir.parent, "cloud_data", "download_info.json"),
        drive_folder_id=drive_folder_id,
    )
    
    required_data = [
        {"drive_name": "bayut_buildings.parquet", "local_path": bayut_buildings_path},
        {"drive_name": "pf_buildings.pkl", "local_path": pf_buildings_path},
        {"drive_name": "building_names_with_geo.pkl", "local_path": mapping_path},
        {"drive_name": "connection_IDS.pkl", "local_path": connection_ids_path}
    ]
    
    manager.download_files(required_data)
    #%%
    import pandas as pd
    
    pf_locations_df = pd.read_pickle(pf_buildings_path)
    bayut_buildings_df = pd.read_parquet(bayut_buildings_path, engine='pyarrow')
    #%%
    pf_locations_df['categories'] = pf_locations_df['categories'].astype(str)
    #%%
    pf_locations_df[
        (pf_locations_df['url_city_slug'] == 'dubai') &
        ~(
                pf_locations_df['categories'].str.lower().str.contains("villa", na=False) |
                pf_locations_df['url_slug'].str.lower().str.contains("villa", na=False) |
                pf_locations_df['name'].str.lower().str.contains("villa", na=False)
        )
        ].shape[0]
    #%%
    pf_locations_df[pf_locations_df['url_city_slug'] == 'dubai'].shape
    #%% md
    ## Блок для улучшенного нечеткого сравнения
    #%% md
    ## Класс и функции для поиска связок
    #%% md
    Класс для вычисления расстояний с мемоизацией
    #%%
    import numpy as np
    from functools import lru_cache
    
    
    class DistanceChecker:
        EARTH_RADIUS = 6371000  # Радиус Земли в метрах
    
        @staticmethod
        @lru_cache(maxsize=None)
        def haversine_distance(lat1, lon1, lat2, lon2):
            """
            Вычисляет расстояние между двумя точками на сфере (метры).
            """
            lat1, lon1, lat2, lon2 = map(np.radians, [lat1, lon1, lat2, lon2])
            dlat = lat2 - lat1
            dlon = lon2 - lon1
            a = np.sin(dlat / 2) ** 2 + np.cos(lat1) * np.cos(lat2) * np.sin(dlon / 2) ** 2
            c = 2 * np.arctan2(np.sqrt(a), np.sqrt(1 - a))
            return round(DistanceChecker.EARTH_RADIUS * c, 2)
    
        def are_points_within_distance(self, lat1, lon1, lat2, lon2, max_distance):
            """
            Проверяет, находятся ли две точки на расстоянии <= max_distance (метры).
            """
            return self.haversine_distance(lat1, lon1, lat2, lon2) <= max_distance
    #%% md
    Функция для обработки алиасов, чтобы перебирать все возможные варианты названия, если имеются
    #%%
    
    
    
    def preprocess_aliases(df):
        """Приводит колонку 'aliases' к списку в DataFrame"""
    
        def parse_aliases(alias_str):
            return ast.literal_eval(alias_str) if alias_str.startswith("[") else [alias_str]
    
        if 'aliases' in df.columns:
            df['aliases'] = df['aliases'].apply(parse_aliases)
    #%%
    preprocess_aliases(bayut_buildings_df)
    #%%
    bayut_buildings_df.loc[bayut_buildings_df['externalID'] == 5003].iloc[0].to_dict()['aliases']
    #%%
    checker = DistanceChecker()
    
    # Проверяем расстояние между двумя точками
    lat1, lon1 = 52.5200, 13.4050  # Берлин
    lat2, lon2 = 52.5205, 13.4065  # Рядом с Берлином
    max_dist = 200  # 200 метров
    
    result = checker.are_points_within_distance(lat1, lon1, lat2, lon2, max_dist)
    print(result)  # True или False
    #%% md
    ## Формируем LAT LON двух таблиц.
    #%% md
    Сформируем таблицу для зданий PF
    #%%
    pf_towers_df = pf_locations_df[
        (pf_locations_df["location_type"] == "TOWER") | (pf_locations_df["location_type"] == "SUBCOMMUNITY")].copy()
    #%%
    pf_towers_df['lat'] = pf_towers_df['coordinates'].apply(lambda x: x['lat'])
    pf_towers_df['lon'] = pf_towers_df['coordinates'].apply(lambda x: x['lon'])
    #%%
    pf_locations_df[pf_locations_df['location_type'].isna()].shape[0]
    #%%
    pf_dubai_towers_locations_df = pf_locations_df[
        (pf_locations_df['url_city_slug'] == 'dubai') &
        ((pf_locations_df["location_type"] == "TOWER") | (pf_locations_df["location_type"] == "SUBCOMMUNITY"))
        ].copy()
    
    pf_dubai_towers_locations_df['district'] = pf_dubai_towers_locations_df['path_name'].apply(
        lambda x: x.split(", ")[1] if x.count(", ") >= 2 else x.split(", ", 1)[-1] if ", " in x else None
    )
    
    pf_dubai_towers_locations_df['lat'] = pf_dubai_towers_locations_df['coordinates'].apply(lambda x: x['lat'])
    pf_dubai_towers_locations_df['lon'] = pf_dubai_towers_locations_df['coordinates'].apply(lambda x: x['lon'])
    #%% md
    Сформируем таблицу для районов PF
    #%%
    pf_areas_df = pf_locations_df[
        (pf_locations_df["location_type"] == "COMMUNITY") | (pf_locations_df["location_type"] == "CITY")].copy()
    
    pf_areas_df['lat'] = pf_areas_df['coordinates'].apply(lambda x: x['lat'])
    pf_areas_df['lon'] = pf_areas_df['coordinates'].apply(lambda x: x['lon'])
    #%%
    bayut_buildings_df[['lat', 'lon']] = bayut_buildings_df['gps'].str.split(', ', expand=True)
    
    bayut_buildings_df['lat'] = bayut_buildings_df['lat'].astype(float)
    bayut_buildings_df['lon'] = bayut_buildings_df['lon'].astype(float)
    #%% md
    Прочитаем таблицу готовых соединений, где 100% совпадение, просто сразу заполним
    #%%
    connection_ids_df = pd.read_pickle(connection_ids_path)
    #%%
    matched_df = connection_ids_df[connection_ids_df['Best_Score'] == 100].copy()
    matched_df.shape
    #%%
    from tqdm import tqdm
    
    if 'PFID' not in bayut_buildings_df.columns:
        bayut_buildings_df['PFID'] = None
    
    for index, row in tqdm(bayut_buildings_df.iterrows(), total=len(bayut_buildings_df), desc="Filling PF buildings"):
        if pd.notna(row['PFID']):
            continue
    
        res = matched_df.loc[matched_df['externalID'] == row['externalID']]
    
        if not res.empty:
            target_data = res.iloc[0]
            bayut_buildings_df.at[index, 'PFID'] = target_data['PFID']
            bayut_buildings_df.at[index, 'PF_Name'] = target_data['pf_name']
            bayut_buildings_df.at[index, 'Best_Score'] = 100
            bayut_buildings_df.at[index, 'location_type'] = target_data['pf_location_type']
    #%%
    bayut_buildings_df.head()
    #%% md
    ### Заполнение районов
    #%% md
    Прочитаем таблицу, в которой уже обработаны все связки по зданиям и добавим районы
    #%%
    import pandas as pd
    from pathlib import Path
    import shutil
    
    target_true_connect_path = Path("../data/true_connect_bayut_to_pf_full.pkl")
    source_true_connect_path = Path("../cloud_data/connection_IDS.pkl")
    #%%
    if not target_true_connect_path.exists():
        print(f"{target_true_connect_path} не найден, копирую из {source_true_connect_path}")
        target_true_connect_path.parent.mkdir(parents=True, exist_ok=True)
        shutil.copy(source_true_connect_path, target_true_connect_path)
    #%%
    true_connect_df = pd.read_pickle(target_true_connect_path)
    #%%
    true_connect_df.head()
    #%%
    if 'lat' not in true_connect_df.columns:
        true_connect_df = true_connect_df.merge(
            bayut_buildings_df[['externalID', 'gps']],
            on='externalID',
            how='left'
        )
    
        true_connect_df[['lat', 'lon']] = (
            true_connect_df['gps']
            .str.replace(" ", "", regex=False)
            .str.split(",", expand=True)
            .astype(float)
        )
    #%%
    if 'lat' not in bayut_buildings_df.columns:
        bayut_buildings_df[['lat', 'lon']] = (
            bayut_buildings_df['gps']
            .str.replace(" ", "", regex=False)
            .str.split(",", expand=True)
            .astype(float)
        )
    #%%
    print("Уникальны ли PFID:")
    true_connect_df[true_connect_df['PFID'].notna()].shape[0] == len(true_connect_df['PFID'].unique()) - 1
    #%%
    true_connect_df[true_connect_df['PFID'].notna()].shape[0]
    #%% md
    ## Ручное добавление PFID
    #%%
    manual_bayut_to_pf_mapping = {
        9959: 13552, 9958: 13564, 14726: 3521, 14725: 3520, 14719: 3514, 14720: 3515, 14723: 3518, 14721: 3516,
        14722: 3517, 7034: 657, 12241: 10148, 11211: 4583, 11210: 4582, 11209: 4581, 11208: 4580, 8030: 3480,
        9454: 4509, 10130: 8865, 8290: 4511, 9050: 8980, 9430: 9094, 5107: 2382, 17711: 14148, 5731: 2524,
        7026: 2525, 12073: 539, 10175: 2478, 5729: 2433, 5150: 2398, 12952: 13713, 9651: 547, 13220: 2463,
        8592: 629, 7081: 2703, 12649: 13803, 17321: 12279, 17322: 14571, 17320: 12282, 11590: 2528, 11591: 2529,
        5741: 522, 8624: 2473, 6914: 536, 9129: 8327, 16460: 12023, 5710: 2431, 11881: 11418, 8832: 2500,
        12320: 10944,
        9105: 9173,
        17945: 13012,
        9292: 2734,
        21010: 2759,
        12908: 2739,
        12906: 2737,
        12910: 2742,
        12909: 2740,
        17870: 12177,
        15273: 10261,
        12260: 9530,
        20262: 13876,
        7789: 8937,
        8137: 630,
        14085: 10660,
        15265: 11748,
        15107: 14738,
        7919: 4426,
        12692: 8456,
        12666: 8606,
        10008: 2092,
        10142: 10325,
        9851: 9070,
        9845: 4506,
        19830: 10439,
        12505: 10111,
        10032: 2099,
        8086: 11430,
        15298: 11780,
        12732: 9832,
        12892: 10905,
        14325: 11348,
        17226: 12644,
        8295: 4358,
        8936: 1216,
        20372: 14157,
        14286: 8511,
        16473: 12159,
        20031: 13104,
        8294: 12415,
        17743: 12775,
        19567: 13229,
        17004: 4254,
        17410: 12731,
        18018: 13235,
        16840: 12313,
        20525: 14301,
        7879: 2278,
        20029: 13736,
        20522: 14483,
        5064: 3165,
        14348: 13002,
        7994: 3117,
        7404: 3104,
        7916: 3058,
        17448: 12948,
        17449: 12952,
        12196: 9414,
        12609: 9413,
        12321: 9410,
        12367: 9409,
        12383: 9418,
        9649: 3769,
        7539: 3767,
        8448: 3817,
        9876: 3836,
        9877: 3835,
        9878: 3834,
        19202: 3729,
        15861: 11947,
        13926: 11339,
        8607: 2142,
        8605: 2144,
        7332: 2139,
        8606: 2126,
        20382: 13924,
        9223: 11874,
        20290: 13849,
        12874: 14039,
        20653: 14233,
        7968: 9178,
        9121: 90,
        14907: 65,
        12025: 2120,
        12026: 2121,
        17088: 12474,
        19762: 13952,
        11663: 4591,
        10059: 4355,
        17121: 12512,
        18020: 13237,
        18019: 13236,
        18973: 12076,
        8048: 10926,
        10215: 2085,
        13570: 10408,
        11183: 387,
        20614: 10114,
        14357: 10865,
        11187: 10927,
        13370: 10820,
        19029: 14431,
        14270: 11382,
        17074: 12607,
        8285: 545,
        9069: 2557,
        9070: 2558,
        9071: 2559,
        12597: 10528,
        5116: 2442,
        14649: 11913,
        17814: 13063,
        17816: 13062,
        17815: 13061,
        19476: 14083,
        20613: 14333,
        7368: 2701,
        8417: 619,
        9287: 621,
        9593: 4419,
        14521: 11427,
        5346: 4430,
        20974: 14617,
        19781: 13930,
        19782: 14022,
        19783: 14699,
        11147: 4423,
        12727: 2260,
        14088: 12128,
        15239: 11857,
        17679: 10049,
        17680: 9295,
        21284: 10924,
        17678: 10050,
        12521: 2183,
        17332: 12624,
        14498: 10463,
        14499: 10736,
        20484: 14054,
        17783: 12968,
        18399: 12884,
        14436: 21076,
        21125: 14625,
        20564: 14181,
        21012: 12906,
        14771: 10663,
        21100: 14716,
        13785: 2190,
        16912: 12364,
        16911: 12365,
        16554: 12056,
        9510: 468,
        9514: 467,
        9512: 466,
        9511: 465,
        7628: 1146,
        16812: 12648,
        9201: 9041,
        9419: 6727,
        9421: 58,
        8231: 3699,
        17093: 3641,
        18987: 13615,
        18314: 14474,
        13123: 3141,
        14112: 9645,
        15073: 11604,
        13337: 746,
        16710: 12225,
        9965: 14365,
        9488: 45,
        8268: 8422,
        12144: 9955,
        12186: 10199,
        19924: 13639,
        14785: 11507,
        20216: 10605,
        15530: 11844,
        9276: 3210,
        17509: 12820,
        15847: 9118,
        18628: 13414,
        19501: 13413,
        15149: 11338,
        19375: 2147,
        7637: 698,
        8988: 708,
        11412: 8900,
        11481: 12570,
        17765: 13260,
        20122: 14370,
        20123: 14344,
        20121: 14327,
        20126: 14290,
        20125: 14313,
        20118: 14316,
        20124: 14314,
        20120: 14350,
        20119: 14410,
        19603: 13570,
        12529: 10438,
        13997: 11757,
        16803: 12480,
        11836: 9688,
        17823: 12831,
        11669: 9347,
        11668: 9346,
        11667: 9345,
        9110: 9730
    }
    #%%
    len(manual_bayut_to_pf_mapping)
    #%%
    def get_pf_object_from_external_id(external_id: int, manual_bayut_to_pf_mapping: dict) -> dict:
        res_obj = {
            'PFID': None,
            'PF_Name': None,
            'Best_Score': None,
            'location_type': None
        }
    
        if external_id not in manual_bayut_to_pf_mapping:
            return res_obj
    
        target_row = pf_locations_df[pf_locations_df['id'] == manual_bayut_to_pf_mapping[external_id]]
    
        if target_row.empty:
            return res_obj
    
        target_data = target_row.iloc[0].to_dict()
        res_obj['PFID'] = manual_bayut_to_pf_mapping[external_id]
        res_obj['PF_Name'] = target_data['name']
        res_obj['Best_Score'] = 100
        res_obj['location_type'] = target_data['location_type']
    
        return res_obj
    #%%
    import pandas as pd
    
    pf_locations_df = pd.read_pickle(pf_buildings_path)
    #%%
    bayut_buildings_df[bayut_buildings_df['externalID'] == 14907].iloc[0].to_dict()['name']
    #%%
    get_pf_object_from_external_id(14907, manual_bayut_to_pf_mapping)
    #%%
    true_connect_df.head()
    #%%
    true_connect_df[true_connect_df['PFID'].notna()].shape
    #%%
    for index, row in true_connect_df.iterrows():
        external_id = row['externalID']
        if external_id in manual_bayut_to_pf_mapping.keys():
            target_pf_obj = get_pf_object_from_external_id(external_id, manual_bayut_to_pf_mapping)
            for key in target_pf_obj:
                true_connect_df.at[index, key] = target_pf_obj[key]
    #%%
    true_connect_df[true_connect_df['PFID'].notna()].shape
    #%%
    true_connect_df[(true_connect_df["Best_Score"] > 0)].shape
    #%%
    true_connect_df['PFID'] = true_connect_df['PFID'].astype("Int64")
    #%%
    list_skip_pf_ids = true_connect_df[true_connect_df['PFID'].notna()]['PFID'].to_list()
    #%%
    len(list_skip_pf_ids)
    #%% md
    ## Отобразим на карте здания, которые не сошлись
    #%%
    import folium
    #%%
    true_connect_df.shape
    #%%
    mean_lat = true_connect_df['lat'].mean()
    mean_lon = true_connect_df['lon'].mean()
    #%%
    # --- ДОБАВЬТЕ ЭТИ ИМПОРТЫ ---
    from folium import FeatureGroup, LayerControl
    
    # --- КОНЕЦ ДОБАВЛЕНИЯ ---
    
    m = folium.Map(location=[mean_lat, mean_lon], zoom_start=12)
    
    # --- ДОБАВЬТЕ СОЗДАНИЕ ГРУПП СЛОЕВ ---
    # Группа для объединенных точек Bayut (зеленые круги и маркеры)
    fg_combined_bayut = FeatureGroup(name='✅ Bayut: Объединенные', show=True)
    # Группа для НЕ объединенных точек Bayut (только маркеры)
    fg_uncombined_bayut = FeatureGroup(name='❓ Bayut: Не объединенные', show=True)
    # Группа для НЕ объединенных точек PF
    fg_uncombined_pf = FeatureGroup(name='❓ PF: Не объединенные', show=True)
    # --- КОНЕЦ ДОБАВЛЕНИЯ ---
    #%%
    # for _, row in bayut_buildings_df.iterrows():
    #     lat, lon = row['lat'], row['lon']
    #     name = row['name']
    #     type = row['type']
    #     external_id = row['externalID']
    #     # pf_name = row['PF_Name']
    #     aliases = row['aliases']
    #     pf_id = row['PFID']
    #     pf_name = row['PF_Name']
    #     # best_score = row.get('Best_Score', 0)
    #
    #     if type == 'condo-building':
    #         color = 'green'
    #     elif type == 'neighbourhood':
    #         color = 'orange'
    #     else:
    #         color = 'yellow'
    #
    #     if pd.notna(pf_id):
    #         folium.Circle(
    #             location=[lat, lon],
    #             radius=10,  # Размер круга можно настроить
    #             color='green',
    #             fill=True,
    #             fill_color='green',
    #             fill_opacity=0.3
    #         ).add_to(m)
    #
    #     folium.CircleMarker(
    #         location=[lat, lon],
    #         radius=5,
    #         color=color,
    #         fill=True,
    #         fill_color=color,
    #         fill_opacity=0.7,
    #         popup=folium.Popup(
    #             f"{name} (type: {type}) (externalID: {external_id})(Aliases: {aliases}) (PFID: {pf_id}) (PF_Name: {pf_name}",
    #             parse_html=True)
    #     ).add_to(m)
    #%%
    # --- ИЗМЕНЕННЫЙ ЦИКЛ ---
    print("Обработка точек Bayut для карты...")  # Добавим вывод
    for _, row in tqdm(bayut_buildings_df.iterrows(), total=len(bayut_buildings_df),
                       desc="Bayut Map Points"):  # Добавим tqdm
        # Проверка на NaN координаты перед использованием
        if pd.isna(row['lat']) or pd.isna(row['lon']):
            continue  # Пропускаем строки с невалидными координатами
    
        lat, lon = row['lat'], row['lon']
        name = row.get('name', 'N/A')  # Безопасное получение значений
        type_ = row.get('type', 'N/A')  # Используем ваше имя переменной type_
        external_id = row.get('externalID', 'N/A')
        aliases = row.get('aliases', [])
        pf_id = row.get('PFID', None)  # Используем get для обработки возможного отсутствия колонки
        pf_name = row.get('PF_Name', 'N/A')
    
        # Определяем цвет маркера Bayut (независимо от объединения)
        if type_ == 'condo-building':
            bayut_marker_color = 'green'
        elif type_ == 'neighbourhood':
            bayut_marker_color = 'orange'
        else:
            bayut_marker_color = 'yellow'
    
        # Формируем popup
        popup_html = f"""
        <b>Bayut Info</b><br>
        --------------------<br>
        <b>Name:</b> {name}<br>
        <b>Type:</b> {type_}<br>
        <b>ExtID:</b> {external_id}<br>
        <b>Aliases:</b> {aliases}<br>
        <b>PFID:</b> {pf_id if pd.notna(pf_id) else '<i>None</i>'}<br>
        <b>PF Name:</b> {pf_name if pd.notna(pf_name) else '<i>None</i>'}
        """
        popup = folium.Popup(popup_html, max_width=350)
    
        # --- Логика добавления в группы ---
        if pd.notna(pf_id):
            # === Это объединенная точка ===
            tooltip_combined = f"✅ Объед.: {name} (Тип: {type_})"
            # 1. Добавляем большой зеленый круг в группу объединенных
            folium.Circle(
                location=[lat, lon],
                radius=10,  # Размер круга можно настроить
                color='green',  # Всегда зеленый для объединенных
                fill=True,
                fill_color='green',
                fill_opacity=0.3,
                tooltip=tooltip_combined  # Добавим tooltip
            ).add_to(fg_combined_bayut)  # <-- ИЗМЕНЕНО: Добавляем в группу fg_combined_bayut
    
            # 2. Добавляем основной маркер (с цветом по типу) в группу объединенных
            folium.CircleMarker(
                location=[lat, lon],
                radius=5,
                color=bayut_marker_color,  # Цвет по типу
                fill=True,
                fill_color=bayut_marker_color,
                fill_opacity=0.7,
                popup=popup,  # Popup добавляем сюда
                tooltip=tooltip_combined  # И сюда
            ).add_to(fg_combined_bayut)  # <-- ИЗМЕНЕНО: Добавляем в группу fg_combined_bayut
        else:
            # === Это НЕ объединенная точка Bayut ===
            tooltip_uncombined = f"❓ Bayut Необъед.: {name} (Тип: {type_})"
            # Добавляем ТОЛЬКО маркер (с цветом по типу) в группу необъединенных Bayut
            folium.CircleMarker(
                location=[lat, lon],
                radius=5,
                color=bayut_marker_color,  # Цвет по типу
                fill=True,
                fill_color=bayut_marker_color,
                fill_opacity=0.7,
                popup=popup,  # Popup добавляем сюда
                tooltip=tooltip_uncombined
            ).add_to(fg_uncombined_bayut)  # <-- ИЗМЕНЕНО: Добавляем в группу fg_uncombined_bayut
    # --- КОНЕЦ ИЗМЕНЕННОГО ЦИКЛА ---
    #%%
    pf_dubai_locations_df = pf_locations_df[
        (pf_locations_df['url_city_slug'] == 'dubai')
    ].copy()
    #%%
    cnt = 0
    for index, row in pf_dubai_locations_df.iterrows():
        pf_id = row['id']
        if pf_id not in set(true_connect_df['PFID']):
            cnt += 1
    cnt
    #%%
    pf_dubai_locations_df['lat'] = pf_dubai_locations_df['coordinates'].apply(lambda x: x['lat'])
    pf_dubai_locations_df['lon'] = pf_dubai_locations_df['coordinates'].apply(lambda x: x['lon'])
    #%%
    # for _, row in pf_dubai_locations_df.iterrows():
    #     pf_id = row['id']
    #     if pf_id not in set(true_connect_df['PFID']):
    #         lat, lon = row['lat'], row['lon']
    #         name = row['name']
    #         location_type = row['location_type']
    #
    #         if location_type == 'TOWER':
    #             color = 'purple'
    #         elif location_type == 'SUBCOMMUNITY':
    #             color = 'gray'
    #         else:
    #             color = 'black'
    #
    #         folium.CircleMarker(
    #             location=[lat, lon],
    #             radius=5,
    #             color=color,
    #             fill=True,
    #             fill_color=color,
    #             fill_opacity=0.7,
    #             popup=folium.Popup(f"{name} (pf_id: {pf_id}) (type: {location_type})", parse_html=True)
    #         ).add_to(m)
    #%%
    # --- ИЗМЕНЕННЫЙ ЦИКЛ ---
    print("Обработка точек PF для карты...")  # Добавим вывод
    # Получаем set ID перед циклом для эффективности
    # Убедимся, что PFID в true_connect_df - это числа (если возможно)
    try:
        matched_pf_ids_set = set(true_connect_df['PFID'].dropna().astype(int))
    except ValueError:
        print("Предупреждение: Не удалось привести PFID к int, возможны ошибки сравнения.")
        matched_pf_ids_set = set(true_connect_df['PFID'].dropna())
    
    # Убедимся что координаты в pf_dubai_locations_df существуют и не NaN
    if 'lat' not in pf_dubai_locations_df.columns or pf_dubai_locations_df['lat'].isnull().any():
        pf_dubai_locations_df['lat'] = pf_dubai_locations_df['coordinates'].apply(
            lambda x: x.get('lat') if isinstance(x, dict) else None)
    if 'lon' not in pf_dubai_locations_df.columns or pf_dubai_locations_df['lon'].isnull().any():
        pf_dubai_locations_df['lon'] = pf_dubai_locations_df['coordinates'].apply(
            lambda x: x.get('lon') if isinstance(x, dict) else None)
    pf_map_df = pf_dubai_locations_df.dropna(subset=['lat', 'lon', 'id']).copy()
    # Приводим ID к int для сравнения с set
    try:
        pf_map_df['id'] = pf_map_df['id'].astype(int)
    except ValueError:
        print("Предупреждение: Не удалось привести id PF к int, возможны ошибки сравнения.")
        pf_map_df['id'] = pd.to_numeric(pf_map_df['id'], errors='coerce')
        pf_map_df.dropna(subset=['id'], inplace=True)  # Удаляем те, что не смогли привестись
    
    for _, row in tqdm(pf_map_df.iterrows(), total=len(pf_map_df), desc="PF Map Points"):
        pf_id = row['id']
        # Проверяем, что этот ID *не* входит в число объединенных
        if pf_id not in matched_pf_ids_set:
            lat, lon = row['lat'], row['lon']
            name = row.get('name', 'N/A')
            location_type = row.get('location_type', 'N/A')
    
            if location_type == 'TOWER':
                color = 'purple'
            elif location_type == 'SUBCOMMUNITY':
                color = 'gray'
            else:
                color = 'black'
    
            popup_html = f"<b>PF Info (Необъед.)</b><br>Name: {name}<br>PF_ID: {pf_id}<br>Type: {location_type}"
            popup = folium.Popup(popup_html, max_width=300)
            tooltip = f"❓ PF Необъед.: {name} (PFID: {pf_id})"
    
            folium.CircleMarker(
                location=[lat, lon],
                radius=5,
                color=color,
                fill=True,
                fill_color=color,
                fill_opacity=0.7,
                popup=popup,  # Используем созданный popup
                tooltip=tooltip  # Добавляем tooltip
            ).add_to(fg_uncombined_pf)  # <-- ИЗМЕНЕНО: Добавляем в группу fg_uncombined_pf
    # --- КОНЕЦ ИЗМЕНЕННОГО ЦИКЛА ---
    #%%
    # import folium
    # from folium import LayerControl
    # import pandas as pd
    #
    # # Создаём FeatureGroup для объединённых точек с show=False, чтобы слой был скрыт по умолчанию
    # combined_points_layer = folium.FeatureGroup(name='Объединенные точки', show=False)
    #
    # # Добавляем круги в FeatureGroup
    # for _, row in bayut_buildings_df.iterrows():
    #     lat, lon = row['lat'], row['lon']
    #     name = row['name']
    #     type_ = row['type']  # избегаем name "type"
    #     pf_id = row['PFID']
    #     pf_name = row['PF_Name']
    #
    #     if type_ == 'condo-building':
    #         color = 'green'
    #     elif type_ == 'neighbourhood':
    #         color = 'orange'
    #     else:
    #         color = 'yellow'
    #
    #     if pd.notna(pf_id):
    #         folium.Circle(
    #             location=[lat, lon],
    #             radius=10,
    #             color=color,
    #             fill=True,
    #             fill_color=color,
    #             fill_opacity=0.3,
    #             popup=folium.Popup(f"Name: {name}<br>Type: {type_}<br>PF_Name: {pf_name}", max_width=250)
    #         ).add_to(combined_points_layer)
    #
    # # Добавляем слой с объединёнными точками на карту
    # combined_points_layer.add_to(m)
    #
    # # Добавляем контрол слоев (если он ещё не добавлен)
    # LayerControl(collapsed=False).add_to(m)
    #
    # # Добавляем кастомную легенду (HTML)
    # legend_html = '''
    # <div style="
    #     position: fixed;
    #     bottom: 50px; left: 50px; width: 200px; height: 110px;
    #     background-color: white;
    #     border:2px solid grey;
    #     z-index:9999;
    #     font-size:14px;
    #     opacity: 0.85;
    #     padding: 10px;
    #     box-shadow: 3px 3px 6px rgba(0,0,0,0.3);
    #     ">
    #     <b>Легенда</b><br>
    #     <i style="background:green; width: 15px; height: 15px; float: left; margin-right: 8px; opacity: 0.7;"></i> Condo-building<br>
    #     <i style="background:orange; width: 15px; height: 15px; float: left; margin-right: 8px; opacity: 0.7;"></i> Neighbourhood<br>
    #     <i style="background:yellow; width: 15px; height: 15px; float: left; margin-right: 8px; opacity: 0.7;"></i> Other<br>
    # </div>
    # '''
    # m.get_root().html.add_child(folium.Element(legend_html))
    #%%
    # m.save("../output/buildings_map_comparison_bayut_and_pf.html")
    #%%
    # --- ДОБАВЬТЕ ЭТОТ КОД ПЕРЕД m.save() ---
    print("Добавление слоев и контроллера на карту...")
    # Добавляем созданные группы на карту m
    fg_combined_bayut.add_to(m)
    fg_uncombined_bayut.add_to(m)
    fg_uncombined_pf.add_to(m)
    
    # Добавляем контроллер слоев ПОСЛЕ добавления групп
    LayerControl(collapsed=False).add_to(m)
    # --- КОНЕЦ ДОБАВЛЕНИЯ ---
    
    # Ваша строка сохранения карты остается здесь же
    m.save("../output/buildings_map_comparison_bayut_and_pf_LAYERED.html")  # Можно изменить имя файла
    print("Карта с контролем слоев сохранена.")
    #%% md
    Добавление легенды с возможностью скрытия объединенных точек
    #%%
    # import pandas as pd
    # import folium
    # from folium import LayerControl
    #
    # # Создаем FeatureGroup для объединенных точек
    # combined_points_layer = folium.FeatureGroup(name='Объединенные точки', show=True)
    #
    # # Добавляем круги
    # for _, row in bayut_buildings_df.iterrows():
    #     lat, lon = row['lat'], row['lon']
    #     name = row['name']
    #     type_ = row['type']  # type — зарезервированное слово, лучше так
    #     pf_id = row['PFID']
    #     pf_name = row['PF_Name']
    #
    #     if type_ == 'condo-building':
    #         color = 'green'
    #     elif type_ == 'neighbourhood':
    #         color = 'orange'
    #     else:
    #         color = 'yellow'
    #
    #     if pd.notna(pf_id):
    #         folium.Circle(
    #             location=[lat, lon],
    #             radius=10,
    #             color=color,
    #             fill=True,
    #             fill_color=color,
    #             fill_opacity=0.3,
    #             popup=folium.Popup(f"Name: {name}<br>Type: {type_}<br>PF_Name: {pf_name}", max_width=250)
    #         ).add_to(combined_points_layer)
    #
    # # Добавляем слой на существующую карту
    # combined_points_layer.add_to(m)
    #
    # # Добавляем контрол слоев (сразу после добавления слоев)
    # LayerControl(collapsed=False).add_to(m)
    #
    # # Кастомная HTML-легенда
    # legend_html = '''
    # <div style="
    #     position: fixed;
    #     bottom: 50px; left: 50px; width: 180px; height: 110px;
    #     background-color: white;
    #     border:2px solid grey;
    #     z-index:9999;
    #     font-size:14px;
    #     opacity: 0.8;
    #     padding: 10px;
    #     ">
    #     <b>Легенда</b><br>
    #     <i style="background:green; width: 15px; height: 15px; float: left; margin-right: 8px; opacity: 0.7;"></i> Condo-building<br>
    #     <i style="background:orange; width: 15px; height: 15px; float: left; margin-right: 8px; opacity: 0.7;"></i> Neighbourhood<br>
    #     <i style="background:yellow; width: 15px; height: 15px; float: left; margin-right: 8px; opacity: 0.7;"></i> Other<br>
    # </div>
    # '''
    #
    # m.get_root().html.add_child(folium.Element(legend_html))
    
    #%%
    m.save("../output/map_with_hide.html")
    #%%
    print(pf_dubai_locations_df['polygon'].apply(type).value_counts())
    
    #%%
    import ast
    
    
    def safe_convert_to_list(value):
        if isinstance(value, list):
            return value
        elif isinstance(value, str):
            try:
                return ast.literal_eval(value)  # Преобразует строку в список, если это возможно
            except (ValueError, SyntaxError):
                return None  # Если преобразование не удалось, заменяем на None
        else:
            return None  # Для всех остальных типов данных
    
    
    pf_dubai_locations_df['polygon'] = pf_dubai_locations_df['polygon'].apply(safe_convert_to_list)
    pf_dubai_locations_df['polygon'] = pf_dubai_locations_df['polygon'].apply(lambda x: x if isinstance(x, list) else [])
    pf_dubai_locations_df['polygon'] = pf_dubai_locations_df['polygon'].apply(lambda x: x if isinstance(x, list) else None)
    
    #%%
    print(pf_dubai_locations_df['polygon'].apply(type).value_counts())
    
    #%%
    bayut_buildings_df.head(1)
    #%%
    pf_dubai_locations_df.head(1)
    #%%
    true_connect_df.head(1)
    #%%
    # Путь для сохранения файлов
    output_dir = this_file_dir.parent / "../source"
    
    # Убедимся, что папка существует
    output_dir.mkdir(parents=True, exist_ok=True)
    
    # Сохраняем DataFrame'ы в формате Parquet
    bayut_buildings_parquet_path = output_dir / "bayut_buildings_df.parquet"
    pf_dubai_locations_parquet_path = output_dir / "pf_dubai_locations_df.parquet"
    true_connect_parquet_path = output_dir / "true_connect_df.parquet"
    
    print("Сохранение DataFrame'ов в формате Parquet...")
    
    # Сохраняем bayut_buildings_df
    bayut_buildings_df.to_parquet(bayut_buildings_parquet_path, engine="pyarrow")
    print(f"bayut_buildings_df сохранен в {bayut_buildings_parquet_path}")
    
    # Сохраняем pf_dubai_locations_df
    pf_dubai_locations_df.to_parquet(pf_dubai_locations_parquet_path, engine="pyarrow")
    print(f"pf_dubai_locations_df сохранен в {pf_dubai_locations_parquet_path}")
    
    # Сохраняем true_connect_df
    true_connect_df.to_parquet(true_connect_parquet_path, engine="pyarrow")
    print(f"true_connect_df сохранен в {true_connect_parquet_path}")
    
    print("Сохранение завершено.")
    

    Нужно его переделать так, что бы в результате получались три датафрейма: bayut_buildings_df pf_buildings_df true_connected_df

  2. Gemini Assistant
    #%% md
    # Ноутбук для улучшения связки BAYUT и PF
    #%%
    import os
    import sys
    from pathlib import Path
    import pandas as pd
    import numpy as np
    import ast
    import folium
    from folium import FeatureGroup, LayerControl
    from tqdm import tqdm
    from functools import lru_cache
    import shutil
    
    
    from modules.google_drive_manager import GoogleDriveManager
    
    this_file_dir = Path.cwd()
    sys.path.append(str(this_file_dir.parent))
    
    bayut_buildings_path = os.path.join(this_file_dir.parent, "cloud_data", "bayut_buildings.parquet")
    pf_buildings_path = os.path.join(this_file_dir.parent, "cloud_data", "pf_buildings.parquet")
    mapping_path = os.path.join(this_file_dir.parent, "cloud_data", "building_names_with_geo.pkl")
    connection_ids_path = os.path.join(this_file_dir.parent, "cloud_data", "connection_IDS.pkl")
    
    building_desc_info_path = os.path.join(
        this_file_dir.parent, "cloud_data", "building_description_info.json"
    )
    
    # LOADING DATA FROM DRIVE
    drive_folder_id = "19i06Z_3xx1FDmIXWpijG_Jzm37eteySd"
    manager = GoogleDriveManager(
        google_service_account_file=os.path.join(
            this_file_dir.parent, "cloud_auth_file", "dubai-estate-7831cb4f621b.json"
        ),
        download_info_file_path=os.path.join(this_file_dir.parent, "cloud_data", "download_info.json"),
        drive_folder_id=drive_folder_id,
    )
    
    required_data = [
        {"drive_name": "bayut_buildings.parquet", "local_path": bayut_buildings_path},
        {"drive_name": "pf_buildings.pkl", "local_path": pf_buildings_path},
        {"drive_name": "building_names_with_geo.pkl", "local_path": mapping_path},
        {"drive_name": "connection_IDS.pkl", "local_path": connection_ids_path}
    ]
    
    manager.download_files(required_data)
    #%%
    
    pf_locations_df = pd.read_pickle(pf_buildings_path)
    bayut_buildings_df = pd.read_parquet(bayut_buildings_path, engine='pyarrow')
    #%%
    pf_locations_df['categories'] = pf_locations_df['categories'].astype(str)
    #%%
    pf_locations_df[
        (pf_locations_df['url_city_slug'] == 'dubai') &
        ~(
                pf_locations_df['categories'].str.lower().str.contains("villa", na=False) |
                pf_locations_df['url_slug'].str.lower().str.contains("villa", na=False) |
                pf_locations_df['name'].str.lower().str.contains("villa", na=False)
        )
        ].shape[0]
    #%%
    pf_locations_df[pf_locations_df['url_city_slug'] == 'dubai'].shape
    #%% md
    ## Блок для улучшенного нечеткого сравнения
    #%% md
    ## Класс и функции для поиска связок
    #%% md
    Класс для вычисления расстояний с мемоизацией
    #%%
    
    
    class DistanceChecker:
        EARTH_RADIUS = 6371000  # Радиус Земли в метрах
    
        @staticmethod
        @lru_cache(maxsize=None)
        def haversine_distance(lat1, lon1, lat2, lon2):
            """
            Вычисляет расстояние между двумя точками на сфере (метры).
            """
            lat1, lon1, lat2, lon2 = map(np.radians, [lat1, lon1, lat2, lon2])
            dlat = lat2 - lat1
            dlon = lon2 - lon1
            a = np.sin(dlat / 2) ** 2 + np.cos(lat1) * np.cos(lat2) * np.sin(dlon / 2) ** 2
            c = 2 * np.arctan2(np.sqrt(a), np.sqrt(1 - a))
            return round(DistanceChecker.EARTH_RADIUS * c, 2)
    
        def are_points_within_distance(self, lat1, lon1, lat2, lon2, max_distance):
            """
            Проверяет, находятся ли две точки на расстоянии <= max_distance (метры).
            """
            return self.haversine_distance(lat1, lon1, lat2, lon2) <= max_distance
    #%% md
    Функция для обработки алиасов, чтобы перебирать все возможные варианты названия, если имеются
    #%%
    
    
    def preprocess_aliases(df):
        """Приводит колонку 'aliases' к списку в DataFrame"""
    
        def parse_aliases(alias_str):
            return ast.literal_eval(alias_str) if alias_str.startswith("[") else [alias_str]
    
        if 'aliases' in df.columns:
            df['aliases'] = df['aliases'].apply(parse_aliases)
    #%%
    preprocess_aliases(bayut_buildings_df)
    #%%
    bayut_buildings_df.loc[bayut_buildings_df['externalID'] == 5003].iloc[0].to_dict()['aliases']
    #%%
    checker = DistanceChecker()
    
    # Проверяем расстояние между двумя точками
    lat1, lon1 = 52.5200, 13.4050  # Берлин
    lat2, lon2 = 52.5205, 13.4065  # Рядом с Берлином
    max_dist = 200  # 200 метров
    
    result = checker.are_points_within_distance(lat1, lon1, lat2, lon2, max_dist)
    print(result)  # True или False
    #%% md
    ## Формируем LAT LON двух таблиц.
    #%% md
    Сформируем таблицу для зданий PF
    #%%
    pf_towers_df = pf_locations_df[
        (pf_locations_df["location_type"] == "TOWER") | (pf_locations_df["location_type"] == "SUBCOMMUNITY")].copy()
    #%%
    pf_towers_df['lat'] = pf_towers_df['coordinates'].apply(lambda x: x['lat'])
    pf_towers_df['lon'] = pf_towers_df['coordinates'].apply(lambda x: x['lon'])
    #%%
    pf_locations_df[pf_locations_df['location_type'].isna()].shape[0]
    #%%
    pf_dubai_towers_locations_df = pf_locations_df[
        (pf_locations_df['url_city_slug'] == 'dubai') &
        ((pf_locations_df["location_type"] == "TOWER") | (pf_locations_df["location_type"] == "SUBCOMMUNITY"))
        ].copy()
    
    pf_dubai_towers_locations_df['district'] = pf_dubai_towers_locations_df['path_name'].apply(
        lambda x: x.split(", ")[1] if x.count(", ") >= 2 else x.split(", ", 1)[-1] if ", " in x else None
    )
    
    pf_dubai_towers_locations_df['lat'] = pf_dubai_towers_locations_df['coordinates'].apply(lambda x: x['lat'])
    pf_dubai_towers_locations_df['lon'] = pf_dubai_towers_locations_df['coordinates'].apply(lambda x: x['lon'])
    #%% md
    Сформируем таблицу для районов PF
    #%%
    pf_areas_df = pf_locations_df[
        (pf_locations_df["location_type"] == "COMMUNITY") | (pf_locations_df["location_type"] == "CITY")].copy()
    
    pf_areas_df['lat'] = pf_areas_df['coordinates'].apply(lambda x: x['lat'])
    pf_areas_df['lon'] = pf_areas_df['coordinates'].apply(lambda x: x['lon'])
    #%%
    bayut_buildings_df[['lat', 'lon']] = bayut_buildings_df['gps'].str.split(', ', expand=True)
    
    bayut_buildings_df['lat'] = bayut_buildings_df['lat'].astype(float)
    bayut_buildings_df['lon'] = bayut_buildings_df['lon'].astype(float)
    #%% md
    Прочитаем таблицу готовых соединений, где 100% совпадение, просто сразу заполним
    #%%
    connection_ids_df = pd.read_pickle(connection_ids_path)
    #%%
    matched_df = connection_ids_df[connection_ids_df['Best_Score'] == 100].copy()
    matched_df.shape
    #%%
    from tqdm import tqdm
    
    if 'PFID' not in bayut_buildings_df.columns:
        bayut_buildings_df['PFID'] = None
    
    for index, row in tqdm(bayut_buildings_df.iterrows(), total=len(bayut_buildings_df), desc="Filling PF buildings"):
        if pd.notna(row['PFID']):
            continue
    
        res = matched_df.loc[matched_df['externalID'] == row['externalID']]
    
        if not res.empty:
            target_data = res.iloc[0]
            bayut_buildings_df.at[index, 'PFID'] = target_data['PFID']
            bayut_buildings_df.at[index, 'PF_Name'] = target_data['pf_name']
            bayut_buildings_df.at[index, 'Best_Score'] = 100
            bayut_buildings_df.at[index, 'location_type'] = target_data['pf_location_type']
    #%%
    bayut_buildings_df.head()
    #%% md
    ### Заполнение районов
    #%% md
    Прочитаем таблицу, в которой уже обработаны все связки по зданиям и добавим районы
    #%%
    target_true_connect_path = Path("../data/true_connect_bayut_to_pf_full.pkl")
    source_true_connect_path = Path("../cloud_data/connection_IDS.pkl")
    #%%
    if not target_true_connect_path.exists():
        print(f"{target_true_connect_path} не найден, копирую из {source_true_connect_path}")
        target_true_connect_path.parent.mkdir(parents=True, exist_ok=True)
        shutil.copy(source_true_connect_path, target_true_connect_path)
    #%%
    true_connect_df = pd.read_pickle(target_true_connect_path)
    #%%
    true_connect_df.head()
    #%%
    if 'lat' not in true_connect_df.columns:
        true_connect_df = true_connect_df.merge(
            bayut_buildings_df[['externalID', 'gps']],
            on='externalID',
            how='left'
        )
    
        true_connect_df[['lat', 'lon']] = (
            true_connect_df['gps']
            .str.replace(" ", "", regex=False)
            .str.split(",", expand=True)
            .astype(float)
        )
    #%%
    if 'lat' not in bayut_buildings_df.columns:
        bayut_buildings_df[['lat', 'lon']] = (
            bayut_buildings_df['gps']
            .str.replace(" ", "", regex=False)
            .str.split(",", expand=True)
            .astype(float)
        )
    #%%
    print("Уникальны ли PFID:")
    true_connect_df[true_connect_df['PFID'].notna()].shape[0] == len(true_connect_df['PFID'].unique()) - 1
    #%%
    true_connect_df[true_connect_df['PFID'].notna()].shape[0]
    #%% md
    ## Ручное добавление PFID
    #%%
    manual_bayut_to_pf_mapping = {
        9959: 13552, 9958: 13564, 14726: 3521, 14725: 3520, 14719: 3514, 14720: 3515, 14723: 3518, 14721: 3516,
        14722: 3517, 7034: 657, 12241: 10148, 11211: 4583, 11210: 4582, 11209: 4581, 11208: 4580, 8030: 3480,
        9454: 4509, 10130: 8865, 8290: 4511, 9050: 8980, 9430: 9094, 5107: 2382, 17711: 14148, 5731: 2524,
        7026: 2525, 12073: 539, 10175: 2478, 5729: 2433, 5150: 2398, 12952: 13713, 9651: 547, 13220: 2463,
        8592: 629, 7081: 2703, 12649: 13803, 17321: 12279, 17322: 14571, 17320: 12282, 11590: 2528, 11591: 2529,
        5741: 522, 8624: 2473, 6914: 536, 9129: 8327, 16460: 12023, 5710: 2431, 11881: 11418, 8832: 2500,
        12320: 10944,
        9105: 9173,
        17945: 13012,
        9292: 2734,
        21010: 2759,
        12908: 2739,
        12906: 2737,
        12910: 2742,
        12909: 2740,
        17870: 12177,
        15273: 10261,
        12260: 9530,
        20262: 13876,
        7789: 8937,
        8137: 630,
        14085: 10660,
        15265: 11748,
        15107: 14738,
        7919: 4426,
        12692: 8456,
        12666: 8606,
        10008: 2092,
        10142: 10325,
        9851: 9070,
        9845: 4506,
        19830: 10439,
        12505: 10111,
        10032: 2099,
        8086: 11430,
        15298: 11780,
        12732: 9832,
        12892: 10905,
        14325: 11348,
        17226: 12644,
        8295: 4358,
        8936: 1216,
        20372: 14157,
        14286: 8511,
        16473: 12159,
        20031: 13104,
        8294: 12415,
        17743: 12775,
        19567: 13229,
        17004: 4254,
        17410: 12731,
        18018: 13235,
        16840: 12313,
        20525: 14301,
        7879: 2278,
        20029: 13736,
        20522: 14483,
        5064: 3165,
        14348: 13002,
        7994: 3117,
        7404: 3104,
        7916: 3058,
        17448: 12948,
        17449: 12952,
        12196: 9414,
        12609: 9413,
        12321: 9410,
        12367: 9409,
        12383: 9418,
        9649: 3769,
        7539: 3767,
        8448: 3817,
        9876: 3836,
        9877: 3835,
        9878: 3834,
        19202: 3729,
        15861: 11947,
        13926: 11339,
        8607: 2142,
        8605: 2144,
        7332: 2139,
        8606: 2126,
        20382: 13924,
        9223: 11874,
        20290: 13849,
        12874: 14039,
        20653: 14233,
        7968: 9178,
        9121: 90,
        14907: 65,
        12025: 2120,
        12026: 2121,
        17088: 12474,
        19762: 13952,
        11663: 4591,
        10059: 4355,
        17121: 12512,
        18020: 13237,
        18019: 13236,
        18973: 12076,
        8048: 10926,
        10215: 2085,
        13570: 10408,
        11183: 387,
        20614: 10114,
        14357: 10865,
        11187: 10927,
        13370: 10820,
        19029: 14431,
        14270: 11382,
        17074: 12607,
        8285: 545,
        9069: 2557,
        9070: 2558,
        9071: 2559,
        12597: 10528,
        5116: 2442,
        14649: 11913,
        17814: 13063,
        17816: 13062,
        17815: 13061,
        19476: 14083,
        20613: 14333,
        7368: 2701,
        8417: 619,
        9287: 621,
        9593: 4419,
        14521: 11427,
        5346: 4430,
        20974: 14617,
        19781: 13930,
        19782: 14022,
        19783: 14699,
        11147: 4423,
        12727: 2260,
        14088: 12128,
        15239: 11857,
        17679: 10049,
        17680: 9295,
        21284: 10924,
        17678: 10050,
        12521: 2183,
        17332: 12624,
        14498: 10463,
        14499: 10736,
        20484: 14054,
        17783: 12968,
        18399: 12884,
        14436: 21076,
        21125: 14625,
        20564: 14181,
        21012: 12906,
        14771: 10663,
        21100: 14716,
        13785: 2190,
        16912: 12364,
        16911: 12365,
        16554: 12056,
        9510: 468,
        9514: 467,
        9512: 466,
        9511: 465,
        7628: 1146,
        16812: 12648,
        9201: 9041,
        9419: 6727,
        9421: 58,
        8231: 3699,
        17093: 3641,
        18987: 13615,
        18314: 14474,
        13123: 3141,
        14112: 9645,
        15073: 11604,
        13337: 746,
        16710: 12225,
        9965: 14365,
        9488: 45,
        8268: 8422,
        12144: 9955,
        12186: 10199,
        19924: 13639,
        14785: 11507,
        20216: 10605,
        15530: 11844,
        9276: 3210,
        17509: 12820,
        15847: 9118,
        18628: 13414,
        19501: 13413,
        15149: 11338,
        19375: 2147,
        7637: 698,
        8988: 708,
        11412: 8900,
        11481: 12570,
        17765: 13260,
        20122: 14370,
        20123: 14344,
        20121: 14327,
        20126: 14290,
        20125: 14313,
        20118: 14316,
        20124: 14314,
        20120: 14350,
        20119: 14410,
        19603: 13570,
        12529: 10438,
        13997: 11757,
        16803: 12480,
        11836: 9688,
        17823: 12831,
        11669: 9347,
        11668: 9346,
        11667: 9345,
        9110: 9730
    }
    #%%
    len(manual_bayut_to_pf_mapping)
    #%%
    def get_pf_object_from_external_id(external_id: int, manual_bayut_to_pf_mapping: dict) -> dict:
        res_obj = {
            'PFID': None,
            'PF_Name': None,
            'Best_Score': None,
            'location_type': None
        }
    
        if external_id not in manual_bayut_to_pf_mapping:
            return res_obj
    
        target_row = pf_locations_df[pf_locations_df['id'] == manual_bayut_to_pf_mapping[external_id]]
    
        if target_row.empty:
            return res_obj
    
        target_data = target_row.iloc[0].to_dict()
        res_obj['PFID'] = manual_bayut_to_pf_mapping[external_id]
        res_obj['PF_Name'] = target_data['name']
        res_obj['Best_Score'] = 100
        res_obj['location_type'] = target_data['location_type']
    
        return res_obj
    #%%
    pf_locations_df = pd.read_pickle(pf_buildings_path)
    #%%
    bayut_buildings_df[bayut_buildings_df['externalID'] == 14907].iloc[0].to_dict()['name']
    #%%
    get_pf_object_from_external_id(14907, manual_bayut_to_pf_mapping)
    #%%
    true_connect_df.head()
    #%%
    true_connect_df[true_connect_df['PFID'].notna()].shape
    #%%
    for index, row in true_connect_df.iterrows():
        external_id = row['externalID']
        if external_id in manual_bayut_to_pf_mapping.keys():
            target_pf_obj = get_pf_object_from_external_id(external_id, manual_bayut_to_pf_mapping)
            for key in target_pf_obj:
                true_connect_df.at[index, key] = target_pf_obj[key]
    #%%
    true_connect_df[true_connect_df['PFID'].notna()].shape
    #%%
    true_connect_df[(true_connect_df["Best_Score"] > 0)].shape
    #%%
    true_connect_df['PFID'] = true_connect_df['PFID'].astype("Int64")
    #%%
    list_skip_pf_ids = true_connect_df[true_connect_df['PFID'].notna()]['PFID'].to_list()
    #%%
    len(list_skip_pf_ids)
    #%%
    
    #%%
    import folium
    #%%
    true_connect_df.shape
    #%%
    mean_lat = true_connect_df['lat'].mean()
    mean_lon = true_connect_df['lon'].mean()
    #%%
    # --- ДОБАВЬТЕ ЭТИ ИМПОРТЫ ---
    
    
    # --- КОНЕЦ ДОБАВЛЕНИЯ ---
    
    m = folium.Map(location=[mean_lat, mean_lon], zoom_start=12)
    
    # --- ДОБАВЬТЕ СОЗДАНИЕ ГРУПП СЛОЕВ ---
    # Группа для объединенных точек Bayut (зеленые круги и маркеры)
    fg_combined_bayut = FeatureGroup(name='✅ Bayut: Объединенные', show=True)
    # Группа для НЕ объединенных точек Bayut (только маркеры)
    fg_uncombined_bayut = FeatureGroup(name='❓ Bayut: Не объединенные', show=True)
    # Группа для НЕ объединенных точек PF
    fg_uncombined_pf = FeatureGroup(name='❓ PF: Не объединенные', show=True)
    # --- КОНЕЦ ДОБАВЛЕНИЯ ---
    #%%
    # for _, row in bayut_buildings_df.iterrows():
    #     lat, lon = row['lat'], row['lon']
    #     name = row['name']
    #     type = row['type']
    #     external_id = row['externalID']
    #     # pf_name = row['PF_Name']
    #     aliases = row['aliases']
    #     pf_id = row['PFID']
    #     pf_name = row['PF_Name']
    #     # best_score = row.get('Best_Score', 0)
    #
    #     if type == 'condo-building':
    #         color = 'green'
    #     elif type == 'neighbourhood':
    #         color = 'orange'
    #     else:
    #         color = 'yellow'
    #
    #     if pd.notna(pf_id):
    #         folium.Circle(
    #             location=[lat, lon],
    #             radius=10,  # Размер круга можно настроить
    #             color='green',
    #             fill=True,
    #             fill_color='green',
    #             fill_opacity=0.3
    #         ).add_to(m)
    #
    #     folium.CircleMarker(
    #         location=[lat, lon],
    #         radius=5,
    #         color=color,
    #         fill=True,
    #         fill_color=color,
    #         fill_opacity=0.7,
    #         popup=folium.Popup(
    #             f"{name} (type: {type}) (externalID: {external_id})(Aliases: {aliases}) (PFID: {pf_id}) (PF_Name: {pf_name}",
    #             parse_html=True)
    #     ).add_to(m)
    #%%
    # --- ИЗМЕНЕННЫЙ ЦИКЛ ---
    print("Обработка точек Bayut для карты...")  # Добавим вывод
    for _, row in tqdm(bayut_buildings_df.iterrows(), total=len(bayut_buildings_df),
                       desc="Bayut Map Points"):  # Добавим tqdm
        # Проверка на NaN координаты перед использованием
        if pd.isna(row['lat']) or pd.isna(row['lon']):
            continue  # Пропускаем строки с невалидными координатами
    
        lat, lon = row['lat'], row['lon']
        name = row.get('name', 'N/A')  # Безопасное получение значений
        type_ = row.get('type', 'N/A')  # Используем ваше имя переменной type_
        external_id = row
    

Responses were produced in different, uncontrolled contexts (system prompts, settings, prior turns). This is not a benchmark.