Skip to content
TheChatArchiveThe Chat Archive
Qwenqwen3-235b-a22b· Alibaba (Qwen)Hugging Face LMArena Human Preference 140K

traindata = loadjson('dataset/train.json') devdata = loadjson('dataset/dev.json')…

From a Chatbot Arena battleThe same person sent this prompt to two anonymous models at once. Compare it with Qwenqwen3-235b-a22b-no-thinking’s answer.
  1. User

    train_data = load_json('dataset/train.json') dev_data = load_json('dataset/dev.json') test_data = load_json('dataset/test.json')

    train_dataset = MedicalNERDataset(train_data, tokenizer, label2id) dev_dataset = MedicalNERDataset(dev_data, tokenizer, label2id) test_dataset = MedicalNERDataset(test_data, tokenizer, label2id)

    train_loader = DataLoader(train_dataset, batch_size=8, shuffle=True) dev_loader = DataLoader(dev_dataset, batch_size=8) test_loader = DataLoader(test_dataset, batch_size=8)import torch from sklearn.metrics import classification_report

    def train_model(model, dataloader, optimizer, device): model.train() total_loss = 0 for batch in dataloader: inputs, labels = batch input_ids, attention_mask = inputs input_ids = input_ids.to(device) attention_mask = attention_mask.to(device) labels = labels.to(device)

        optimizer.zero_grad()
        loss = model(input_ids, attention_mask, labels)
        loss.backward()
        optimizer.step()
        total_loss += loss.item()
    return total_loss / len(dataloader)
    

    def evaluate_model(model, dataloader, device): model.eval() all_preds = [] all_trues = [] with torch.no_grad(): for i, batch in enumerate(dataloader): inputs, labels = batch input_ids, attention_mask = inputs input_ids = input_ids.to(device) attention_mask = attention_mask.to(device) labels = labels.to(device)

            preds = model(input_ids, attention_mask)
    
            for pred, label, mask in zip(preds, labels, attention_mask):
                active_indices = mask.bool()
                filtered_pred = [p for p, m in zip(pred, active_indices) if m]
                filtered_label = [l.item() for l, m in zip(label, active_indices) if m]
    
                assert len(filtered_pred) == len(filtered_label), \
                    f"Length mismatch: pred={len(filtered_pred)}, label={len(filtered_label)}"
    
                all_preds.extend(filtered_pred)
                all_trues.extend(filtered_label)
    return all_preds, all_trues
    

    def get_classification_report(predictions, true_labels): y_true = [id2label[label] for label in true_labels] y_pred = [id2label[label] for label in predictions]

    report_dict = classification_report(y_true, y_pred, target_names=label2id.keys(), digits=4, output_dict=True)
    return report_dict
    

    def train_and_evaluate_model(model, model_name, train_loader, dev_loader, optimizer, device, num_epochs=5): """训练并评估模型""" performance_records[model_name] = []

    for epoch in range(num_epochs):
        train_loss = train_model(model, train_loader, optimizer, device)
        preds, trues = evaluate_model(model, dev_loader, device)
        report = get_classification_report(preds, trues)
        performance_records[model_name].append(report['weighted avg'])
        
        print(f"Epoch {epoch + 1}/{num_epochs} - Loss: {train_loss:.4f}")torch.save(model_bert_crf.state_dict(), 'bert_crf_model.pth')
    

    torch.save(model_lstm_crf.state_dict(), 'bilstm_crf_model.pth') torch.save(model_lstm_attn_crf.state_dict(), 'bilstm_attn_crf_model.pth') print("Models saved successfully!")三个模型训练和评估时是否使用测试集数据,如未使用则用测试集数据对三个模型进行评估并可视化对比

  2. Qwen Assistant

    根据代码分析,三个模型在训练和评估过程中未使用测试集数据,仅使用了开发集(dev)进行验证。为完善模型评估,需补充对测试集的评估流程,并进行可视化对比。以下是完整的解决方案:


    1. 测试集评估代码补充

    # 加载保存的模型权重(需先定义模型结构)
    model_bert_crf.load_state_dict(torch.load('bert_crf_model.pth'))
    model_lstm_crf.load_state_dict(torch.load('bilstm_crf_model.pth'))
    model_lstm_attn_crf.load_state_dict(torch.load('bilstm_attn_crf_model.pth'))
    
    # 将模型移动到设备(如GPU)
    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
    model_bert_crf.to(device)
    model_lstm_crf.to(device)
    model_lstm_attn_crf.to(device)
    
    # 定义模型名称与模型的映射
    models = {
        "BERT-CRF": model_bert_crf,
        "BiLSTM-CRF": model_lstm_crf,
        "BiLSTM-Attn-CRF": model_lstm_attn_crf
    }
    
    # 在测试集上评估所有模型
    test_results = {}
    for model_name, model in models.items():
        preds, trues = evaluate_model(model, test_loader, device)
        report = get_classification_report(preds, trues)
        test_results[model_name] = report
        print(f"Test Results for {model_name}:")
        print(classification_report([id2label[label] for label in trues], 
                                    [id2label[label] for label in preds],
                                    target_names=label2id.keys()))
    

    2. 可视化对比代码

    使用 matplotlib 和 seaborn 可视化各模型的加权平均 F1 分数、精确率和召回率:

    import matplotlib.pyplot as plt
    import seaborn as sns
    import pandas as pd
    
    # 提取各模型的加权平均指标
    metrics = ["precision", "recall", "f1-score"]
    model_names = list(test_results.keys())
    results = {metric: [test_results[model]["weighted avg"][metric] for model in model_names] 
               for metric in metrics}
    
    # 转换为 DataFrame
    df = pd.DataFrame(results, index=model_names)
    
    # 绘制柱状图
    plt.figure(figsize=(10, 6))
    df.plot(kind='bar', rot=0, figsize=(12, 8))
    plt.title("Model Performance on Test Set (Weighted Average)")
    plt.ylabel("Score")
    plt.ylim(0.8, 1.0)
    plt.legend(loc='lower right')
    plt.grid(axis='y', linestyle='--', alpha=0.7)
    plt.tight_layout()
    plt.savefig("model_comparison.png")
    plt.show()
    

    3. 结果分析

    • 测试集评估:上述代码会输出每个模型在测试集上的分类报告,包括精确率(Precision)、召回率(Recall)、F1 分数(F1-Score)和样本支持数(Support)。
    • 可视化对比:生成的柱状图将直观展示三个模型在 加权平均指标 上的差异,便于快速比较模型性能。

    4. 关键注意事项

    1. 模型定义一致性:确保在加载模型权重前,已正确定义模型结构(如 BertCRFModel、BiLSTMCRF 等)。
    2. 设备匹配:模型加载后需通过 .to(device) 移动到 GPU/CPU。
    3. 标签映射正确性:确保 id2label 字典与训练时一致,避免标签错位。
    4. 结果解释:重点关注 加权平均(weighted avg) 指标,尤其在类别不平衡的数据集中更具代表性。

    通过以上步骤,您可以完整评估三个模型在测试集上的表现,并通过可视化对比选择最优模型。

Responses were produced in different, uncontrolled contexts (system prompts, settings, prior turns). This is not a benchmark.