Сделай из этого дага простой скрипт на python: import json import logging import pandas…
- User
Сделай из этого дага простой скрипт на python:
import json import logging
import pandas as pd import numpy as np
from pathlib import Path from typing import Any, Optional, Dict, List
from airflow import DAG
from airflow.operators.python import PythonOperator from airflow.providers.http.hooks.http import HttpHook
from configs import settings from plugins.search_indices.extractors import CsvFileExtractor
logger = logging.getLogger(file)
def transfer_brand_transliteration_data_dag( dag: DAG, default_args: Dict[str, Any] ) -> PythonOperator: return PythonOperator( task_id='transfer_brand_transliterations_from_producers_feed', dag=dag, python_callable=transfer_brand_transliteration_data )
def transfer_brand_transliteration_data(): producers_feed_fp = Path(settings.PRODUCERS_FEED_FILEPATH) if not producers_feed_fp.exists(): raise FileNotFoundError(f'Feed not found: {str(producers_feed_fp)}')
brand_transliterations = CsvFileExtractor( filepath=producers_feed_fp, required_fields=['name', 'name_ru'] ) exported_data = transform_brand_transliterations(brand_transliterations()) hook = HttpHook(http_conn_id='synonyms_service') hook.run( endpoint=settings.API_ENDPOINT_BRAND_TRANSLITERATIONS, headers={'Content-Type': 'application/json; charset=utf-8'}, data=json.dumps(exported_data) )def _compile_transliteration_mapping(data: pd.Series) -> Optional[Dict[str, Any]]: brand = data['name'].strip() if not brand: logger.warning('empty brand value') return
transliterations = set(w.strip() for w in data['name_ru'].split(',')) if brand in transliterations: transliterations.remove(brand) if not transliterations: logger.warning('empty transliterations value for brand "%s"', brand) return return {'brand': brand, 'transliterations': list(transliterations)}def transform_brand_transliterations(df: pd.DataFrame) -> List[Dict[str, Any]]: return ( df # find and drop empty strings with multiple spaces .replace(r'^\s+$', np.nan, regex=True).dropna() # prepare mapping for export .apply(_compile_transliteration_mapping, axis=1) .dropna() .tolist() )
- Assistant
Вот простой скрипт на Python, основанный на данном коде:
import json import logging import pandas as pd import numpy as np from pathlib import Path from typing import Any, Optional, Dict, List from airflow.providers.http.hooks.http import HttpHook def transfer_brand_transliteration_data(): logger = logging.getLogger(__file__) producers_feed_fp = Path(settings.PRODUCERS_FEED_FILEPATH) if not producers_feed_fp.exists(): raise FileNotFoundError(f'Feed not found: {str(producers_feed_fp)}') brand_transliterations = CsvFileExtractor( filepath=producers_feed_fp, required_fields=['name', 'name_ru'] ) exported_data = transform_brand_transliterations(brand_transliterations()) hook = HttpHook(http_conn_id='synonyms_service') hook.run( endpoint=settings.API_ENDPOINT_BRAND_TRANSLITERATIONS, headers={'Content-Type': 'application/json; charset=utf-8'}, data=json.dumps(exported_data) ) def _compile_transliteration_mapping(data: pd.Series) -> Optional[Dict[str, Any]]: brand = data['name'].strip() if not brand: logger.warning('empty brand value') return transliterations = set(w.strip() for w in data['name_ru'].split(',')) if brand in transliterations: transliterations.remove(brand) if not transliterations: logger.warning('empty transliterations value for brand "%s"', brand) return return {'brand': brand, 'transliterations': list(transliterations)} def transform_brand_transliterations(df: pd.DataFrame) -> List[Dict[str, Any]]: return ( df .replace(r'^\s+$', np.nan, regex=True).dropna() .apply(_compile_transliteration_mapping, axis=1) .dropna() .tolist() )Вам также может потребоваться импортировать необходимые модули и определить переменную
settings, в которой содержатся необходимые настройки.
Other models, same or similar prompt
Responses were produced in different, uncontrolled contexts (system prompts, settings, prior turns). This is not a benchmark.