首先给定一个文件,我们的目的是从恢复中断的训练,我们需要加载最新的checkpoint开始:import os import glob import torch import…
- User
首先给定一个文件,我们的目的是从恢复中断的训练,我们需要加载最新的checkpoint开始:import os import glob import torch import traceback from lib.train.admin import multigpu from torch.utils.data.distributed import DistributedSampler
class BaseTrainer: """Base trainer class. Contains functions for training and saving/loading checkpoints. Trainer classes should inherit from this one and overload the train_epoch function."""
def __init__(self, actor, loaders, optimizer, settings, lr_scheduler=None): """ args: actor - The actor for training the network loaders - list of dataset loaders, e.g. [train_loader, val_loader]. In each epoch, the trainer runs one epoch for each loader. optimizer - The optimizer used for training, e.g. Adam settings - Training settings lr_scheduler - Learning rate scheduler """ self.actor = actor self.optimizer = optimizer self.lr_scheduler = lr_scheduler self.loaders = loaders self.update_settings(settings) self.epoch = 0 self.stats = {} self.device = getattr(settings, 'device', None) if self.device is None: self.device = torch.device("cuda:0" if torch.cuda.is_available() and settings.use_gpu else "cpu") self.actor.to(self.device) self.settings = settings def update_settings(self, settings=None): """Updates the trainer settings. Must be called to update internal settings.""" if settings is not None: self.settings = settings if self.settings.env.workspace_dir is not None: self.settings.env.workspace_dir = os.path.expanduser(self.settings.env.workspace_dir) '''2021.1.4 New function: specify checkpoint dir''' if self.settings.save_dir is None: self._checkpoint_dir = os.path.join(self.settings.env.workspace_dir, 'checkpoints') else: self._checkpoint_dir = os.path.join(self.settings.save_dir, 'checkpoints') print("checkpoints will be saved to %s" % self._checkpoint_dir) if self.settings.local_rank in [-1, 0]: if not os.path.exists(self._checkpoint_dir): print("Training with multiple GPUs. checkpoints directory doesn't exist. " "Create checkpoints directory") os.makedirs(self._checkpoint_dir) else: self._checkpoint_dir = None def train(self, max_epochs, load_latest=False, fail_safe=True, load_previous_ckpt=False, distill=False): """Do training for the given number of epochs. args: max_epochs - Max number of training epochs, load_latest - Bool indicating whether to resume from latest epoch. fail_safe - Bool indicating whether the training to automatically restart in case of any crashes. """ epoch = -1 num_tries = 1 for i in range(num_tries): try: if load_latest: self.load_checkpoint() if load_previous_ckpt: directory = '{}/{}'.format(self._checkpoint_dir, self.settings.project_path_prv) self.load_state_dict(directory) if distill: directory_teacher = '{}/{}'.format(self._checkpoint_dir, self.settings.project_path_teacher) self.load_state_dict(directory_teacher, distill=True) for epoch in range(self.epoch+1, max_epochs+1): self.epoch = epoch self.train_epoch() if self.lr_scheduler is not None: if self.settings.scheduler_type != 'cosine': self.lr_scheduler.step() else: self.lr_scheduler.step(epoch - 1) # only save the last 10 checkpoints save_every_epoch = getattr(self.settings, "save_every_epoch", False) save_epochs = [79, 159, 239] if epoch > (max_epochs - 1) or save_every_epoch or epoch % 10 == 0 or epoch in save_epochs or epoch > (max_epochs - 3): # if epoch > (max_epochs - 10) or save_every_epoch or epoch % 100 == 0: if self._checkpoint_dir: if self.settings.local_rank in [-1, 0]: self.save_checkpoint() except: print('Training crashed at epoch {}'.format(epoch)) if fail_safe: self.epoch -= 1 load_latest = True print('Traceback for the error!') print(traceback.format_exc()) print('Restarting training from last epoch ...') else: raise print('Finished training!') def train_epoch(self): raise NotImplementedError def save_checkpoint(self): """Saves a checkpoint of the network and other variables.""" net = self.actor.net.module if multigpu.is_multi_gpu(self.actor.net) else self.actor.net actor_type = type(self.actor).__name__ net_type = type(net).__name__ state = { 'epoch': self.epoch, 'actor_type': actor_type, 'net_type': net_type, 'net': net.state_dict(), 'net_info': getattr(net, 'info', None), 'constructor': getattr(net, 'constructor', None), 'optimizer': self.optimizer.state_dict(), 'stats': self.stats, 'settings': self.settings } directory = '{}/{}'.format(self._checkpoint_dir, self.settings.project_path) print(directory) if not os.path.exists(directory): print("directory doesn't exist. creating...") os.makedirs(directory) # First save as a tmp file tmp_file_path = '{}/{}_ep{:04d}.tmp'.format(directory, net_type, self.epoch) torch.save(state, tmp_file_path) file_path = '{}/{}_ep{:04d}.pth.tar'.format(directory, net_type, self.epoch) # Now rename to actual checkpoint. os.rename seems to be atomic if files are on same filesystem. Not 100% sure os.rename(tmp_file_path, file_path) def load_checkpoint(self, checkpoint = None, fields = None, ignore_fields = None, load_constructor = False): """Loads a network checkpoint file. Can be called in three different ways: load_checkpoint(): Loads the latest epoch from the workspace. Use this to continue training. load_checkpoint(epoch_num): Loads the network at the given epoch number (int). load_checkpoint(path_to_checkpoint): Loads the file from the given absolute path (str). """ net = self.actor.net.module if multigpu.is_multi_gpu(self.actor.net) else self.actor.net actor_type = type(self.actor).__name__ net_type = type(net).__name__ if checkpoint is None: # Load most recent checkpoint checkpoint_list = sorted(glob.glob('{}/{}/{}_ep*.pth.tar'.format(self._checkpoint_dir, self.settings.project_path, net_type))) if checkpoint_list: checkpoint_path = checkpoint_list[-1] else: print('No matching checkpoint file found') return elif isinstance(checkpoint, int): # Checkpoint is the epoch number checkpoint_path = '{}/{}/{}_ep{:04d}.pth.tar'.format(self._checkpoint_dir, self.settings.project_path, net_type, checkpoint) elif isinstance(checkpoint, str): # checkpoint is the path if os.path.isdir(checkpoint): checkpoint_list = sorted(glob.glob('{}/*_ep*.pth.tar'.format(checkpoint))) if checkpoint_list: checkpoint_path = checkpoint_list[-1] else: raise Exception('No checkpoint found') else: checkpoint_path = os.path.expanduser(checkpoint) else: raise TypeError # Load network checkpoint_dict = torch.load(checkpoint_path, map_location='cpu') assert net_type == checkpoint_dict['net_type'], 'Network is not of correct type.' if fields is None: fields = checkpoint_dict.keys() if ignore_fields is None: ignore_fields = ['settings'] # Never load the scheduler. It exists in older checkpoints. ignore_fields.extend(['lr_scheduler', 'constructor', 'net_type', 'actor_type', 'net_info']) # Load all fields for key in fields: if key in ignore_fields: continue if key == 'net': net.load_state_dict(checkpoint_dict[key]) elif key == 'optimizer': self.optimizer.load_state_dict(checkpoint_dict[key]) else: setattr(self, key, checkpoint_dict[key]) # Set the net info if load_constructor and 'constructor' in checkpoint_dict and checkpoint_dict['constructor'] is not None: net.constructor = checkpoint_dict['constructor'] if 'net_info' in checkpoint_dict and checkpoint_dict['net_info'] is not None: net.info = checkpoint_dict['net_info'] # Update the epoch in lr scheduler if 'epoch' in fields: self.lr_scheduler.last_epoch = self.epoch # 2021.1.10 Update the epoch in data_samplers for loader in self.loaders: if isinstance(loader.sampler, DistributedSampler): loader.sampler.set_epoch(self.epoch) return True def load_state_dict(self, checkpoint=None, distill=False): """Loads a network checkpoint file. Can be called in three different ways: load_checkpoint(): Loads the latest epoch from the workspace. Use this to continue training. load_checkpoint(epoch_num): Loads the network at the given epoch number (int). load_checkpoint(path_to_checkpoint): Loads the file from the given absolute path (str). """ if distill: net = self.actor.net_teacher.module if multigpu.is_multi_gpu(self.actor.net_teacher) \ else self.actor.net_teacher else: net = self.actor.net.module if multigpu.is_multi_gpu(self.actor.net) else self.actor.net net_type = type(net).__name__ if isinstance(checkpoint, str): # checkpoint is the path if os.path.isdir(checkpoint): checkpoint_list = sorted(glob.glob('{}/*_ep*.pth.tar'.format(checkpoint))) if checkpoint_list: checkpoint_path = checkpoint_list[-1] else: raise Exception('No checkpoint found') else: checkpoint_path = os.path.expanduser(checkpoint) else: raise TypeError # Load network print("Loading pretrained model from ", checkpoint_path) checkpoint_dict = torch.load(checkpoint_path, map_location='cpu') assert net_type == checkpoint_dict['net_type'], 'Network is not of correct type.' missing_k, unexpected_k = net.load_state_dict(checkpoint_dict["net"], strict=False) print("previous checkpoint is loaded.") print("missing keys: ", missing_k) print("unexpected keys:", unexpected_k) return True - Assistant
基于您提供的代码示例,这个
BaseTrainer类设计用于辅助深度学习模型的训练、保存以及恢复中断的训练过程。以下是对关键组件和功能的概述:初始化 (
__init__)- 初始化
BaseTrainer时,需要传入模型(actor)、数据加载器(loaders)、优化器(optimizer)、设置(settings),以及可选的学习率调度器(lr_scheduler)。 - 根据设置自动确定训练使用的设备(CPU或GPU)。
- 初始化训练状态(例如:当前
epoch和统计数据)
训练 (
train)- 主要训练流程,支持从最新的checkpoint恢复训练(
load_latest)和在训练中断后自动重新启动(fail_safe)。 - 支持保存最近几个epochs的模型,以便于恢复和评估。
- 通过
train_epoch(应由子类实现)执行每个epoch的训练。
Checkpoint 保存/加载
save_checkpoint和load_checkpoint方法允许你保存和加载训练过程的状态,包括模型参数、优化器状态、以及epoch等信息。load_state_dict方法允许从指定的路径加载旧模型或教师模型,支持微调和知识蒸馏(distill)场景。
其他关键点
update_settings方法用于更新训练设置。- 通过检测是否使用分布式训练并对应地设置数据加载器采样器,来支持分布式训练。
- 模型的设备分配基于配置自动决定,支持使用GPU进行训练(如果可用)。
- 训练过程中,根据设置决定何时保存checkpoint,以及是否按epoch调整学习率等。
用法
这个
BaseTrainer类设计为被继承,继承时需要实现train_epoch方法,该方法定义了单个epoch的训练逻辑。总结
这段代码提供了一个灵活的框架,用于深度学习训练、checkpoint管理和从中断中恢复。通过合理设计设置和调用相应的方法,可以适应多种训练和恢复需求,同时保持代码的可读性和可维护性。
- 初始化
- User
这里的 try: if load_latest: self.load_checkpoint() if load_previous_ckpt: directory = '{}/{}'.format(self._checkpoint_dir, self.settings.project_path_prv) self.load_state_dict(directory) if distill: directory_teacher = '{}/{}'.format(self._checkpoint_dir, self.settings.project_path_teacher) self.load_state_dict(directory_teacher, distill=True) for epoch in range(self.epoch+1, max_epochs+1): self.epoch = epoch
self.train_epoch()分别执行了什么功能?
- Assistant
这段代码位于
train方法中,用于控制模型的训练过程,并包含了从特定状态恢复训练的逻辑。我们来逐一看看每部分执行的功能:1. 加载最新的Checkpoint
if load_latest: self.load_checkpoint()- 如果
load_latest参数为True,这一步会尝试加载最新的checkpoint来恢复训练。把模型、优化器、以及训练进度(比如epoch值)恢复到最近保存的状态。
2. 加载指定的旧Checkpoint
if load_previous_ckpt: directory = '{}/{}'.format(self._checkpoint_dir, self.settings.project_path_prv) self.load_state_dict(directory)- 如果
load_previous_ckpt为True,则会加载一个指定的、较旧的checkpoint来恢复训练或进行其他操作(比如迁移学习等)。通过参数self.settings.project_path_prv指明checkpoint的路径。
3. 加载教师模型的Checkpoint(用于知识蒸馏)
if distill: directory_teacher = '{}/{}'.format(self._checkpoint_dir, self.settings.project_path_teacher) self.load_state_dict(directory_teacher, distill=True)- 如果
distill为True,意味着将进行知识蒸馏训练过程。这一步将会根据配置(self.settings.project_path_teacher)加载教师模型的checkpoint。这里通过distill=True传入load_state_dict方法标明是教师模型,这可能会影响加载逻辑,使其适应知识蒸馏的上下文。
4. 遍历Epochs执行训练
for epoch in range(self.epoch+1, max_epochs+1): self.epoch = epoch self.train_epoch()- 这个循环从当前epoch开始(如果是新训练会从0开始,如果是恢复的训练,则从checkpoint记录的epoch开始),直到到达
max_epochs。 - 对于每个epoch,首先更新
self.epoch属性以反映当前的epoch数,然后调用train_epoch执行实际的训练过程。train_epoch方法应当由子类具体实现,包含了单个epoch中对所有数据的前向传播、损失计算、反向传播等训练步骤。
总结
在这个代码段中,通过条件判断判断恢复模式(恢复最新的、一个特定的checkpoint,或是加载一个教师模型进行知识蒸馏),并在完成恢复设置后执行实际的训练周期。这样的设计方便灵活地从断点继续训练,并支持不同的训练策略。
- 如果
- User
那么我们需要加载最新的checkpoint来恢复训练
- Assistant
要加载最新的checkpoint来恢复训练,您可以确保在调用
train方法时将load_latest参数设置为True。这个参数会触发train方法内部逻辑,去查找并加载保存的最新checkpoint状态,包括模型参数、优化器状态和训练进度等。在您的
BaseTrainer类中,恢复训练的步骤大致如下:-
设置
load_latest为True: 确保在调用train方法时,传递load_latest=True作为参数。 -
自动调用
load_checkpoint方法: 如果load_latest=True,train方法会自动调用load_checkpoint方法来查找并加载最新的checkpoint。 -
执行训练循环: 加载了checkpoint后,训练将从保存的epoch开始,直到达到最大的epoch数。
下面是一个调用
train方法来恢复最新checkpoint训练的示例代码:# 假设您已经创建了BaseTrainer实例,名称为trainer # 设置最大训练epoch数量 max_epochs = 100 # 开始训练,并尝试从最新的checkpoint恢复 trainer.train(max_epochs, load_latest=True)确保训练前您已有至少一个保存的checkpoint文件在您指定的checkpoint目录中。
load_checkpoint方法将按照您定义的逻辑(通常是按照文件名中的epoch数或保存日期)查找最新的checkpoint文件。这种方式,如果训练过程中发生中断(如因为突发的硬件问题、资源限制或其他外部因素),您可以很容易地从最近一次训练的状态继续,而不是从头开始。这是深度学习实践中一项非常有价值的功能,可以节省大量的时间和计算资源。
-
- User
现在有两个文件中有 load_latest,① :import os
loss function related
from lib.utils.box_ops import giou_loss from torch.nn.functional import l1_loss from torch.nn import BCEWithLogitsLoss
train pipeline related
from lib.train.trainers import LTRTrainer
distributed training related
from torch.nn.parallel import DistributedDataParallel as DDP
some more advanced functions
from .base_functions import *
network related
from lib.models.ceutrack import build_ceutrack
forward propagation related
from lib.train.actors import CEUTrackActor
for import modules
import importlib
from ..utils.focal_loss import FocalLoss
def run(settings): settings.description = 'Training script for STARK-S, STARK-ST stage1, and STARK-ST stage2'
# update the default configs with config file if not os.path.exists(settings.cfg_file): raise ValueError("%s doesn't exist." % settings.cfg_file) config_module = importlib.import_module("lib.config.%s.config" % settings.script_name) cfg = config_module.cfg config_module.update_config_from_file(settings.cfg_file) if settings.local_rank in [-1, 0]: print("New configuration is shown below.") for key in cfg.keys(): print("%s configuration:" % key, cfg[key]) print('\n') # update settings based on cfg update_settings(settings, cfg) # Record the training log log_dir = os.path.join(settings.save_dir, 'logs') if settings.local_rank in [-1, 0]: if not os.path.exists(log_dir): os.makedirs(log_dir) settings.log_file = os.path.join(log_dir, "%s-%s.log" % (settings.script_name, settings.config_name)) # Build dataloaders loader_train, loader_val = build_dataloaders(cfg, settings) if "RepVGG" in cfg.MODEL.BACKBONE.TYPE or "swin" in cfg.MODEL.BACKBONE.TYPE or "LightTrack" in cfg.MODEL.BACKBONE.TYPE: cfg.ckpt_dir = settings.save_dir # Create network if settings.script_name == "ceutrack": net = build_ceutrack(cfg) else: raise ValueError("illegal script name") # wrap networks to distributed one net.cuda() if settings.local_rank != -1: # net = torch.nn.SyncBatchNorm.convert_sync_batchnorm(net) # add syncBN converter net = DDP(net, device_ids=[settings.local_rank], find_unused_parameters=True) settings.device = torch.device("cuda:%d" % settings.local_rank) else: settings.device = torch.device("cuda:0") settings.deep_sup = getattr(cfg.TRAIN, "DEEP_SUPERVISION", False) settings.distill = getattr(cfg.TRAIN, "DISTILL", False) settings.distill_loss_type = getattr(cfg.TRAIN, "DISTILL_LOSS_TYPE", "KL") # Loss functions and Actors if settings.script_name == "ceutrack": focal_loss = FocalLoss() objective = {'giou': giou_loss, 'l1': l1_loss, 'focal': focal_loss, 'cls': BCEWithLogitsLoss()} loss_weight = {'giou': cfg.TRAIN.GIOU_WEIGHT, 'l1': cfg.TRAIN.L1_WEIGHT, 'focal': cfg.TRAIN.FOCAL_WEIGHT, 'cls': 1.0} actor = CEUTrackActor(net=net, objective=objective, loss_weight=loss_weight, settings=settings, cfg=cfg) else: raise ValueError("illegal script name") # if cfg.TRAIN.DEEP_SUPERVISION: # raise ValueError("Deep supervision is not supported now.") # Optimizer, parameters, and learning rates optimizer, lr_scheduler = get_optimizer_scheduler(net, cfg) use_amp = getattr(cfg.TRAIN, "AMP", False) trainer = LTRTrainer(actor, [loader_train, loader_val], optimizer, settings, lr_scheduler, use_amp=use_amp) # train processimport osloss function related
from lib.utils.box_ops import giou_loss from torch.nn.functional import l1_loss from torch.nn import BCEWithLogitsLoss
train pipeline related
from lib.train.trainers import LTRTrainer
distributed training related
from torch.nn.parallel import DistributedDataParallel as DDP
some more advanced functions
from .base_functions import *
network related
from lib.models.stark import build_starks, build_starkst from lib.models.stark import build_stark_lightning_x_trt
forward propagation related
from lib.train.actors import STARKLightningXtrtdistillActor
for import modules
import importlib
def build_network(script_name, cfg): # Create network if script_name == "stark_s": net = build_starks(cfg) elif script_name == "stark_st1" or script_name == "stark_st2": net = build_starkst(cfg) elif script_name == "stark_lightning_X_trt": net = build_stark_lightning_x_trt(cfg, phase="train") else: raise ValueError("illegal script name") return net
def run(settings): settings.description = 'Training script for STARK-S, STARK-ST stage1, and STARK-ST stage2'
# update the default configs with config file if not os.path.exists(settings.cfg_file): raise ValueError("%s doesn't exist." % settings.cfg_file) config_module = importlib.import_module("lib.config.%s.config" % settings.script_name) cfg = config_module.cfg config_module.update_config_from_file(settings.cfg_file) if settings.local_rank in [-1, 0]: print("New configuration is shown below.") for key in cfg.keys(): print("%s configuration:" % key, cfg[key]) print('\n') # update the default teacher configs with teacher config file if not os.path.exists(settings.cfg_file_teacher): raise ValueError("%s doesn't exist." % settings.cfg_file_teacher) config_module_teacher = importlib.import_module("lib.config.%s.config" % settings.script_teacher) cfg_teacher = config_module_teacher.cfg config_module_teacher.update_config_from_file(settings.cfg_file_teacher) if settings.local_rank in [-1, 0]: print("New teacher configuration is shown below.") for key in cfg_teacher.keys(): print("%s configuration:" % key, cfg_teacher[key]) print('\n') # update settings based on cfg update_settings(settings, cfg) # Record the training log log_dir = os.path.join(settings.save_dir, 'logs') if settings.local_rank in [-1, 0]: if not os.path.exists(log_dir): os.makedirs(log_dir) settings.log_file = os.path.join(log_dir, "%s-%s.log" % (settings.script_name, settings.config_name)) # Build dataloaders loader_train, loader_val = build_dataloaders(cfg, settings) if "RepVGG" in cfg.MODEL.BACKBONE.TYPE or "swin" in cfg.MODEL.BACKBONE.TYPE: cfg.ckpt_dir = settings.save_dir """turn on the distillation mode""" cfg.TRAIN.DISTILL = True cfg_teacher.TRAIN.DISTILL = True net = build_network(settings.script_name, cfg) net_teacher = build_network(settings.script_teacher, cfg_teacher) # wrap networks to distributed one net.cuda() net_teacher.cuda() net_teacher.eval() if settings.local_rank != -1: net = DDP(net, device_ids=[settings.local_rank], find_unused_parameters=True) net_teacher = DDP(net_teacher, device_ids=[settings.local_rank], find_unused_parameters=True) settings.device = torch.device("cuda:%d" % settings.local_rank) else: settings.device = torch.device("cuda:0") # settings.deep_sup = getattr(cfg.TRAIN, "DEEP_SUPERVISION", False) # settings.distill = getattr(cfg.TRAIN, "DISTILL", False) settings.distill_loss_type = getattr(cfg.TRAIN, "DISTILL_LOSS_TYPE", "L1") # Loss functions and Actors if settings.script_name == "stark_lightning_X_trt": objective = {'giou': giou_loss, 'l1': l1_loss} loss_weight = {'giou': cfg.TRAIN.GIOU_WEIGHT, 'l1': cfg.TRAIN.L1_WEIGHT} actor = STARKLightningXtrtdistillActor(net=net, objective=objective, loss_weight=loss_weight, settings=settings, net_teacher=net_teacher) else: raise ValueError("illegal script name") # Optimizer, parameters, and learning rates optimizer, lr_scheduler = get_optimizer_scheduler(net, cfg) use_amp = getattr(cfg.TRAIN, "AMP", False) trainer = LTRTrainer(actor, [loader_train, loader_val], optimizer, settings, lr_scheduler, use_amp=use_amp) # train process trainer.train(cfg.TRAIN.EPOCH, load_latest=True, fail_safe=True, distill=True),那么现在 trainer.train(cfg.TRAIN.EPOCH, load_latest=True, fail_safe=True) ② : - Assistant
在两个文件中,你展示了调用
train方法的两种不同方式,分别涉及不同的参数。下面是对这两种方式的概述和它们之间的差异:方式 ①
在第一种方式中,
train方法被调用时包含了这样的参数:trainer.train(cfg.TRAIN.EPOCH, load_latest=True, fail_safe=True, distill=True)- load_latest=True:启用从最新的checkpoint恢复训练的功能,即自动查找最新的checkpoint并从那个状态开始训练。
- fail_safe=True:启用失败保护,如果训练过程中出现异常,将从最近的checkpoint重新开始,尝试继续训练。
- distill=True:启用知识蒸馏模式。在这种模式下,训练流程可能会稍有不同,具体依赖于
LTRTrainer或其继承类中train方法的实现细节。这通常涉及到使用一个预训练的"教师"模型去指导或改进另一个"学生"模型的训练过程,以此来传递知识。
方式 ②
在第二种方式中,
train方法仍然被调用了,但参数略有不同:trainer.train(cfg.TRAIN.EPOCH, load_latest=True, fail_safe=True)- load_latest=True:与方式①相同,启用从最新的checkpoint恢复。
- fail_safe=True:与方式①相同,启用失败保护以支持从最近的checkpoint恢复训练。
- 在这种方式中,没有提供
distill=True参数,这意味着不会启用知识蒸馏模式。训练将以常规方式进行,不涉及额外的教师模型来辅助训练过程。
总结
两种方式的主要差异在于是否启用知识蒸馏。使用
distill=True时,会启用额外的逻辑以支持知识蒸馏过程,这可能需要同时处理学生模型和教师模型。未使用distill参数时,表示采用标准训练流程,不涉及蒸馏过程。选择哪种方式取决于你的具体训练目标。如果只是需要从最新的checkpoint恢复训练,并不需要执行知识蒸馏,那么第二种方式(方式 ②)就足够了。如果你的训练流程需要利用到知识蒸馏的优势,那么应该采用第一种方式(方式 ①)。在任何情况下,确保相应的训练基础设施和逻辑已经准备好支持你选择的参数和训练模式。
Other models, same or similar prompt
Responses were produced in different, uncontrolled contexts (system prompts, settings, prior turns). This is not a benchmark.