traindata = loadjson('dataset/train.json') devdata = loadjson('dataset/dev.json')…
- User
train_data = load_json('dataset/train.json') dev_data = load_json('dataset/dev.json') test_data = load_json('dataset/test.json')
train_dataset = MedicalNERDataset(train_data, tokenizer, label2id) dev_dataset = MedicalNERDataset(dev_data, tokenizer, label2id) test_dataset = MedicalNERDataset(test_data, tokenizer, label2id)
train_loader = DataLoader(train_dataset, batch_size=8, shuffle=True) dev_loader = DataLoader(dev_dataset, batch_size=8) test_loader = DataLoader(test_dataset, batch_size=8)import torch from sklearn.metrics import classification_report
def train_model(model, dataloader, optimizer, device): model.train() total_loss = 0 for batch in dataloader: inputs, labels = batch input_ids, attention_mask = inputs input_ids = input_ids.to(device) attention_mask = attention_mask.to(device) labels = labels.to(device)
optimizer.zero_grad() loss = model(input_ids, attention_mask, labels) loss.backward() optimizer.step() total_loss += loss.item() return total_loss / len(dataloader)def evaluate_model(model, dataloader, device): model.eval() all_preds = [] all_trues = [] with torch.no_grad(): for i, batch in enumerate(dataloader): inputs, labels = batch input_ids, attention_mask = inputs input_ids = input_ids.to(device) attention_mask = attention_mask.to(device) labels = labels.to(device)
preds = model(input_ids, attention_mask) for pred, label, mask in zip(preds, labels, attention_mask): active_indices = mask.bool() filtered_pred = [p for p, m in zip(pred, active_indices) if m] filtered_label = [l.item() for l, m in zip(label, active_indices) if m] assert len(filtered_pred) == len(filtered_label), \ f"Length mismatch: pred={len(filtered_pred)}, label={len(filtered_label)}" all_preds.extend(filtered_pred) all_trues.extend(filtered_label) return all_preds, all_truesdef get_classification_report(predictions, true_labels): y_true = [id2label[label] for label in true_labels] y_pred = [id2label[label] for label in predictions]
report_dict = classification_report(y_true, y_pred, target_names=label2id.keys(), digits=4, output_dict=True) return report_dictdef train_and_evaluate_model(model, model_name, train_loader, dev_loader, optimizer, device, num_epochs=5): """训练并评估模型""" performance_records[model_name] = []
for epoch in range(num_epochs): train_loss = train_model(model, train_loader, optimizer, device) preds, trues = evaluate_model(model, dev_loader, device) report = get_classification_report(preds, trues) performance_records[model_name].append(report['weighted avg']) print(f"Epoch {epoch + 1}/{num_epochs} - Loss: {train_loss:.4f}")torch.save(model_bert_crf.state_dict(), 'bert_crf_model.pth')torch.save(model_lstm_crf.state_dict(), 'bilstm_crf_model.pth') torch.save(model_lstm_attn_crf.state_dict(), 'bilstm_attn_crf_model.pth') print("Models saved successfully!")三个模型训练和评估时是否使用测试集数据,如未使用则用测试集数据对三个模型进行评估并可视化对比
Assistant
根据代码分析,三个模型在训练和评估过程中未使用测试集数据,仅使用了开发集(dev)进行验证。为完善模型评估,需补充对测试集的评估流程,并进行可视化对比。以下是完整的解决方案:
1. 测试集评估代码补充
# 加载保存的模型权重(需先定义模型结构) model_bert_crf.load_state_dict(torch.load('bert_crf_model.pth')) model_lstm_crf.load_state_dict(torch.load('bilstm_crf_model.pth')) model_lstm_attn_crf.load_state_dict(torch.load('bilstm_attn_crf_model.pth')) # 将模型移动到设备(如GPU) device = torch.device("cuda" if torch.cuda.is_available() else "cpu") model_bert_crf.to(device) model_lstm_crf.to(device) model_lstm_attn_crf.to(device) # 定义模型名称与模型的映射 models = { "BERT-CRF": model_bert_crf, "BiLSTM-CRF": model_lstm_crf, "BiLSTM-Attn-CRF": model_lstm_attn_crf } # 在测试集上评估所有模型 test_results = {} for model_name, model in models.items(): preds, trues = evaluate_model(model, test_loader, device) report = get_classification_report(preds, trues) test_results[model_name] = report print(f"Test Results for {model_name}:") print(classification_report([id2label[label] for label in trues], [id2label[label] for label in preds], target_names=label2id.keys()))
2. 可视化对比代码
使用
matplotlib和seaborn可视化各模型的加权平均 F1 分数、精确率和召回率:import matplotlib.pyplot as plt import seaborn as sns import pandas as pd # 提取各模型的加权平均指标 metrics = ["precision", "recall", "f1-score"] model_names = list(test_results.keys()) results = {metric: [test_results[model]["weighted avg"][metric] for model in model_names] for metric in metrics} # 转换为 DataFrame df = pd.DataFrame(results, index=model_names) # 绘制柱状图 plt.figure(figsize=(10, 6)) df.plot(kind='bar', rot=0, figsize=(12, 8)) plt.title("Model Performance on Test Set (Weighted Average)") plt.ylabel("Score") plt.ylim(0.8, 1.0) plt.legend(loc='lower right') plt.grid(axis='y', linestyle='--', alpha=0.7) plt.tight_layout() plt.savefig("model_comparison.png") plt.show()
3. 结果分析
- 测试集评估:上述代码会输出每个模型在测试集上的分类报告,包括精确率(Precision)、召回率(Recall)、F1 分数(F1-Score)和样本支持数(Support)。
- 可视化对比:生成的柱状图将直观展示三个模型在 加权平均指标 上的差异,便于快速比较模型性能。
4. 关键注意事项
- 模型定义一致性:确保在加载模型权重前,已正确定义模型结构(如
BertCRFModel、BiLSTMCRF等)。 - 设备匹配:模型加载后需通过
.to(device)移动到 GPU/CPU。 - 标签映射正确性:确保
id2label字典与训练时一致,避免标签错位。 - 结果解释:重点关注 加权平均(weighted avg) 指标,尤其在类别不平衡的数据集中更具代表性。
通过以上步骤,您可以完整评估三个模型在测试集上的表现,并通过可视化对比选择最优模型。
Other models, same or similar prompt
Responses were produced in different, uncontrolled contexts (system prompts, settings, prior turns). This is not a benchmark.