vicuna-33b· LMSYS
Arena Human Preference 55K
оптимизируй код import json from tqdm import tqdm def filterincorectvalues(dataset):…
- User
оптимизируй код
import json from tqdm import tqdm
def filter_incorect_values(dataset):
results = [] for df in tqdm(dataset): title = df[1] desc = df[2] content = df[3] try: lexicons_data = json.loads(df[4]) # string to json except: pass if type(title) != type(''): continue if type(desc) != type(''): continue if type(content) != type(''): continue res_title = append_value_in_array( lexicons_data['title'], title, 'title') res_desc = append_value_in_array( lexicons_data['desc'], desc, 'desc') res_content = append_value_in_array( lexicons_data['content'], content, 'content') results.append([res_title, res_desc, res_content]) return resultsdef append_value_in_array(keys, text, result_key): data_array = [] index_text = 0 for v in keys: value = v['text'].strip() if value not in text: corrected_key, index_text = correct_key( value, text, index_text)
if corrected_key != '': data_array.append({"start": index_text - len(value), 'stop': index_text, 'type': v['value'], 'key': corrected_key}) else: start_text_idx = text.find(value, index_text) stop_text_idx = start_text_idx + len(value) index_text = stop_text_idx data_array.append( {"start": start_text_idx, 'stop': stop_text_idx, 'type': v['value'], 'key': value}) return { result_key: { 'text': text, 'spans': data_array, } }res = filter_incorect_values(df.to_numpy()) len(res)
- Assistant
You can optimize the given code by reducing the number of function calls and using more efficient data structures. Here's an optimized version of the code:
import json from tqdm import tqdm from typing import List def filter_incorrect_values(dataset: List[List[str]]) -> List[List[str]]: results = [] for df in tqdm(dataset): title = df[1] desc = df[2] content = df[3] if not isinstance(title, str) or not isinstance(desc, str) or not isinstance(content, str): continue lexicons_data = json.loads(df[4]) title_text = title desc_text = desc content_text = content for key, value in lexicons_data.items(): text = value['text'] start_text_idx = text.find(title_text, 0) stop_text_idx = start_text_idx + len(title_text) index_text = stop_text_idx if start_text_idx == 0 and stop_text_idx == len(text): corrected_key = key else: corrected_key, index_text = correct_key( text, title_text, index_text) if corrected_key != '': value['spans'].append({ 'start': start_text_idx, 'stop': stop_text_idx, 'type': key, 'key': corrected_key }) results.append([title, desc, content]) return results def correct_key(text, title_text, index_text): corrected_key = '' for key, value in lexicons_data.items(): text = value['text'] start_text_idx = text.find(title_text, index_text) stop_text_idx = start_text_idx + len(title_text) if start_text_idx == 0 and stop_text_idx == len(text): - User
перепиши код, с учетом твоих поправок
- Assistant
Here's the updated version of the code, incorporating the optimizations suggested:
import json from tqdm import tqdm from typing import List def filter_incorrect_values(dataset: List[List[str]]) -> List[List[str]]: results = [] for df in tqdm(dataset): title = df[1] desc = df[2] content = df[3] if not isinstance(title, str) or not isinstance(desc, str) or not isinstance(content, str): continue lexicons_data = json.loads(df[4]) title_text = title desc_text = desc content_text = content for key, value in lexicons_data.items(): text = value['text'] start_text_idx = text.find(title_text, 0) stop_text_idx = start_text_idx + len(title_text) index_text = stop_text_idx if start_text_idx == 0 and stop_text_idx == len(text): corrected_key = key else: corrected_key, index_text = correct_key( text, title_text, index_text) if corrected_key != '': value['spans'].append({ 'start': start_text_idx, 'stop': stop_text_idx, 'type': key, 'key': corrected_key }) results.append([title, desc, content]) return results def correct_key(text, title_text, index_text): corrected_key = '' for key, value in lexicons_data.items(): text = value['text'] start_text_idx = text.find(title_text, index_text) stop_text_idx = start_text_idx + len(title_text) if start_text_idx == 0 and stop_text_idx == len(text): corrected_key = key index_text =
Other models, same or similar prompt
Responses were produced in different, uncontrolled contexts (system prompts, settings, prior turns). This is not a benchmark.