训练代码是:① import os import argparse import random def parseargs(): """ args for training.…
- User
训练代码是:① import os import argparse import random
def parse_args(): """ args for training. """ parser = argparse.ArgumentParser(description='Parse args for training') # for train parser.add_argument('--script', type=str, help='training script name') parser.add_argument('--config', type=str, default='baseline', help='yaml configure file name') parser.add_argument('--save_dir', type=str, help='root directory to save checkpoints, logs, and tensorboard') parser.add_argument('--mode', type=str, choices=["single", "multiple", "multi_node"], default="multiple", help="train on single gpu or multiple gpus") # parser.add_argument('--nproc_per_node', type=int, help="number of GPUs per node") # specify when mode is multiple parser.add_argument('--nproc_per_node', type=int, default=1, help="number of GPUs per node") parser.add_argument('--use_lmdb', type=int, choices=[0, 1], default=0) # whether datasets are in lmdb format parser.add_argument('--script_prv', type=str, help='training script name') parser.add_argument('--config_prv', type=str, default='baseline', help='yaml configure file name') parser.add_argument('--use_wandb', type=int, choices=[0, 1], default=0) # whether to use wandb # for knowledge distillation parser.add_argument('--distill', type=int, choices=[0, 1], default=0) # whether to use knowledge distillation parser.add_argument('--script_teacher', type=str, help='teacher script name') parser.add_argument('--config_teacher', type=str, help='teacher yaml configure file name')
# for multiple machines parser.add_argument('--rank', type=int, help='Rank of the current process.') parser.add_argument('--world-size', type=int, help='Number of processes participating in the job.') parser.add_argument('--ip', type=str, default='127.0.0.1', help='IP of the current rank 0.') parser.add_argument('--port', type=int, default='20000', help='Port of the current rank 0.') args = parser.parse_args() return argsdef main(): args = parse_args() if args.mode == "single": train_cmd = "python lib/train/run_training.py --script %s --config %s --save_dir %s --use_lmdb %d "
"--script_prv %s --config_prv %s --distill %d --script_teacher %s --config_teacher %s --use_wandb %d"
% (args.script, args.config, args.save_dir, args.use_lmdb, args.script_prv, args.config_prv, args.distill, args.script_teacher, args.config_teacher, args.use_wandb) elif args.mode == "multiple": train_cmd = "torchrun --nproc_per_node %d --master_port %d lib/train/run_training.py "
"--script %s --config %s --save_dir %s --use_lmdb %d --script_prv %s --config_prv %s --use_wandb %d "
"--distill %d --script_teacher %s --config_teacher %s"
% (args.nproc_per_node, random.randint(10000, 50000), args.script, args.config, args.save_dir, args.use_lmdb, args.script_prv, args.config_prv, args.use_wandb, args.distill, args.script_teacher, args.config_teacher) elif args.mode == "multi_node": train_cmd = "python -m torch.distributed.launch --nproc_per_node %d --master_addr %s --master_port %d --nnodes %d --node_rank %d lib/train/run_training.py "
"--script %s --config %s --save_dir %s --use_lmdb %d --script_prv %s --config_prv %s --use_wandb %d "
"--distill %d --script_teacher %s --config_teacher %s"
% (args.nproc_per_node, args.ip, args.port, args.world_size, args.rank, args.script, args.config, args.save_dir, args.use_lmdb, args.script_prv, args.config_prv, args.use_wandb, args.distill, args.script_teacher, args.config_teacher) else: raise ValueError("mode should be 'single' or 'multiple'.") os.system(train_cmd)if name == "main": main() 和 ② import os import sys import argparse import importlib import cv2 as cv import torch.backends.cudnn import torch.distributed as dist
import random import numpy as np torch.backends.cudnn.benchmark = False
import _init_paths import lib.train.admin.settings as ws_settings
def init_seeds(seed): random.seed(seed) np.random.seed(seed) torch.manual_seed(seed) torch.cuda.manual_seed(seed) torch.backends.cudnn.deterministic = True torch.backends.cudnn.benchmark = False
def run_training(script_name, config_name, cudnn_benchmark=True, local_rank=-1, save_dir=None, base_seed=None, use_lmdb=False, script_name_prv=None, config_name_prv=None, use_wandb=False, distill=None, script_teacher=None, config_teacher=None): """Run the train script. args: script_name: Name of emperiment in the "experiments/" folder. config_name: Name of the yaml file in the "experiments/<script_name>". cudnn_benchmark: Use cudnn benchmark or not (default is True). """ if save_dir is None: print("save_dir dir is not given. Use the default dir instead.") # This is needed to avoid strange crashes related to opencv cv.setNumThreads(0)
torch.backends.cudnn.benchmark = cudnn_benchmark print('script_name: {}.py config_name: {}.yaml'.format(script_name, config_name)) '''2021.1.5 set seed for different process''' if base_seed is not None: if local_rank != -1: init_seeds(base_seed + local_rank) else: init_seeds(base_seed) settings = ws_settings.Settings() settings.script_name = script_name settings.config_name = config_name settings.project_path = 'train/{}/{}'.format(script_name, config_name) if script_name_prv is not None and config_name_prv is not None: settings.project_path_prv = 'train/{}/{}'.format(script_name_prv, config_name_prv) settings.local_rank = local_rank settings.save_dir = os.path.abspath(save_dir) settings.use_lmdb = use_lmdb prj_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), "../..")) settings.cfg_file = os.path.join(prj_dir, 'experiments/%s/%s.yaml' % (script_name, config_name)) settings.use_wandb = use_wandb if distill: settings.distill = distill settings.script_teacher = script_teacher settings.config_teacher = config_teacher if script_teacher is not None and config_teacher is not None: settings.project_path_teacher = 'train/{}/{}'.format(script_teacher, config_teacher) settings.cfg_file_teacher = os.path.join(prj_dir, 'experiments/%s/%s.yaml' % (script_teacher, config_teacher)) expr_module = importlib.import_module('lib.train.train_script_distill') else: expr_module = importlib.import_module('lib.train.train_script') expr_func = getattr(expr_module, 'run') expr_func(settings)def main(): parser = argparse.ArgumentParser(description='Run a train scripts in train_settings.') parser.add_argument('--script', type=str, required=True, help='Name of the train script.') parser.add_argument('--config', type=str, required=True, help="Name of the config file.") parser.add_argument('--cudnn_benchmark', type=bool, default=True, help='Set cudnn benchmark on (1) or off (0) (default is on).') parser.add_argument('--local_rank', default=-1, type=int, help='node rank for distributed training') parser.add_argument('--save_dir', type=str, help='the directory to save checkpoints and logs') parser.add_argument('--seed', type=int, default=3407, help='seed for random numbers, default=42') parser.add_argument('--use_lmdb', type=int, choices=[0, 1], default=0) # whether datasets are in lmdb format parser.add_argument('--script_prv', type=str, default=None, help='Name of the train script of previous model.') parser.add_argument('--config_prv', type=str, default=None, help="Name of the config file of previous model.") parser.add_argument('--use_wandb', type=int, choices=[0, 1], default=0) # whether to use wandb # for knowledge distillation parser.add_argument('--distill', type=int, choices=[0, 1], default=0) # whether to use knowledge distillation parser.add_argument('--script_teacher', type=str, help='teacher script name') parser.add_argument('--config_teacher', type=str, help='teacher yaml configure file name')
args = parser.parse_args() ############## if 'LOCAL_RANK' in os.environ: args.local_rank = int(os.environ['LOCAL_RANK']) #################### if args.local_rank != -1: dist.init_process_group(backend='nccl') torch.cuda.set_device(args.local_rank) else: torch.cuda.set_device(0) run_training(args.script, args.config, cudnn_benchmark=args.cudnn_benchmark, local_rank=args.local_rank, save_dir=args.save_dir, base_seed=args.seed, use_lmdb=args.use_lmdb, script_name_prv=args.script_prv, config_name_prv=args.config_prv, use_wandb=args.use_wandb, distill=args.distill, script_teacher=args.script_teacher, config_teacher=args.config_teacher)if name == 'main': # os.environ['CUDA_VISIBLE_DEVICES'] = "4" main() ,执行的命令是:torchrun --nproc_per_node=2 tracking/train.py --script ceutrack --config ceutrack_coesot --save_dir ./output --use_wandb 0,但是目前是仅仅在一张卡上执行两个进程,并不是使用两个GPU,然后一个进程一块GPU,现在分析问题出在哪里,然后给出修改方案。
- Assistant
根据你描述的问题,原因可能是由于进程没有正确地绑定到各自的GPU上。你的命令执行了两个进程,但这两个进程可能都被绑定在了同一张GPU上,而不是分别被绑定到两张不同的GPU上。
通常情况下,在多GPU训练时,需要确保每个进程分配到各自的GPU,这样才能正确地利用多块GPU进行训练。你提到的调度是用
torchrun进行的,因此我们需要确保每个进程分配到不同的GPU。在第二段代码的main函数处理中,
torch.distributed.launch模块会根据local_rank来设置CUDA设备。为了确保样的调度逻辑正确,需要确保每个进程的local_rank变量被正确配置并传递,从而每个进程能够正确地使用不同的GPU。修改后的代码
我们可以修改你的第二段代码,确保每个进程读取并设置自己独立的local_rank。如下所示:
import os import sys import argparse import importlib import cv2 as cv import torch.backends.cudnn import torch.distributed as dist import random import numpy as np torch.backends.cudnn.benchmark = False import _init_paths import lib.train.admin.settings as ws_settings def init_seeds(seed): random.seed(seed) np.random.seed(seed) torch.manual_seed(seed) torch.cuda.manual_seed(seed) torch.backends.cudnn.deterministic = True torch.backends.cudnn.benchmark = False def run_training(script_name, config_name, cudnn_benchmark=True, local_rank=-1, save_dir=None, base_seed=None, use_lmdb=False, script_name_prv=None, config_name_prv=None, use_wandb=False, distill=None, script_teacher=None, config_teacher=None): """Run the train script. args: script_name: Name of emperiment in the "experiments/" folder. config_name: Name of the yaml file in the "experiments/<script_name>". cudnn_benchmark: Use cudnn benchmark or not (default is True). """ if save_dir is None: print("save_dir dir is not given. Use the default dir instead.") # This is needed to avoid strange crashes related to opencv cv.setNumThreads(0) torch.backends.cudnn.benchmark = cudnn_benchmark print('script_name: {}.py config_name: {}.yaml'.format(script_name, config_name)) '''2021.1.5 set seed for different process''' if base_seed is not None: if local_rank != -1: init_seeds(base_seed + local_rank) else: init_seeds(base_seed) settings = ws_settings.Settings() settings.script_name = script_name settings.config_name = config_name settings.project_path = 'train/{}/{}'.format(script_name, config_name) if script_name_prv is not None and config_name_prv is not None: settings.project_path_prv = 'train/{}/{}'.format(script_name_prv, config_name_prv) settings.local_rank = local_rank settings.save_dir = os.path.abspath(save_dir) settings.use_lmdb = use_lmdb prj_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), "../..")) settings.cfg_file = os.path.join(prj_dir, 'experiments/%s/%s.yaml' % (script_name, config_name)) settings.use_wandb = use_wandb if distill: settings.distill = distill settings.script_teacher = script_teacher settings.config_teacher = config_teacher if script_teacher is not None and config_teacher is not None: settings.project_path_teacher = 'train/{}/{}'.format(script_teacher, config_teacher) settings.cfg_file_teacher = os.path.join(prj_dir, 'experiments/%s/%s.yaml' % (script_teacher, config_teacher)) expr_module = importlib.import_module('lib.train.train_script_distill') else: expr_module = importlib.import_module('lib.train.train_script') expr_func = getattr(expr_module, 'run') expr_func(settings) def main(): parser = argparse.ArgumentParser(description='Run a train scripts in train_settings.') parser.add_argument('--script', type=str, required=True, help='Name of the train script.') parser.add_argument('--config', type=str, required=True, help="Name of the config file.") parser.add_argument('--cudnn_benchmark', type=bool, default=True, help='Set cudnn benchmark on (1) or off (0) (default is on).') parser.add_argument('--local_rank', default=-1, type=int, help='node rank for distributed training') parser.add_argument('--save_dir', type=str, help='the directory to save checkpoints and logs') parser.add_argument('--seed', type=int, default=3407, help='seed for random numbers, default=42') parser.add_argument('--use_lmdb', type=int, choices=[0, 1], default=0) # whether datasets are in lmdb format parser.add_argument('--script_prv', type=str, default=None, help='Name of the train script of previous model.') parser.add_argument('--config_prv', type=str, default=None, help="Name of the config file of previous model.") parser.add_argument('--use_wandb', type=int, choices=[0, 1], default=0) # whether to use wandb # for knowledge distillation parser.add_argument('--distill', type=int, choices=[0, 1], default=0) # whether to use knowledge distillation parser.add_argument('--script_teacher', type=str, help='teacher script name') parser.add_argument('--config_teacher', type=str, help='teacher yaml configure file name') args = parser.parse_args() ############## if 'LOCAL_RANK' in os.environ: args.local_rank = int(os.environ['LOCAL_RANK']) #################### if args.local_rank != -1: dist.init_process_group(backend='nccl') torch.cuda.set_device(args.local_rank) else: torch.cuda.set_device(0) run_training(args.script, args.config, cudnn_benchmark=args.cudnn_benchmark, local_rank=args.local_rank, save_dir=args.save_dir, base_seed=args.seed, use_lmdb=args.use_lmdb, script_name_prv=args.script_prv, config_name_prv=args.config_prv, use_wandb=args.use_wandb, distill=args.distill, script_teacher=args.script_teacher, config_teacher=args.config_teacher) if __name__ == '__main__': main()这里主要改动在于确保
local_rank被正确读取并设置,从而每个进程绑定到不同的GPU上。并不需要额外的硬编码设置,因为torchrun会负责将local_rank作为环境变量传递,而我们在代码中读取它。确定使用命令
然后,你运行分布式任务的命令应该是:
torchrun --nproc_per_node=2 tracking/train.py --script ceutrack --config ceutrack_coesot --save_dir ./output --use_wandb 0这样应该确保两个进程分别绑定到两个GPU上。这样你就可以充分利用多GPU来加速训练了。
- User
将改动的地方单独写出来
- Assistant
以下是你提供的代码中需要改动的部分,以确保每个进程能够绑定到各自的 GPU 上。
1. 添加 local_rank 环境变量读取代码
在解析命令行参数之后,添加如下代码来读取
local_rank环境变量:if 'LOCAL_RANK' in os.environ: args.local_rank = int(os.environ['LOCAL_RANK'])2. 根据 local_rank 设置 CUDA 设备
在
main函数中,修改以确保 CUDA 设备根据local_rank进行设置:if args.local_rank != -1: dist.init_process_group(backend='nccl') torch.cuda.set_device(args.local_rank) else: torch.cuda.set_device(0)改动后的集中版本
import os import sys import argparse import importlib import cv2 as cv import torch.backends.cudnn import torch.distributed as dist import random import numpy as np torch.backends.cudnn.benchmark = False import _init_paths import lib.train.admin.settings as ws_settings def init_seeds(seed): random.seed(seed) np.random.seed(seed) torch.manual_seed(seed) torch.cuda.manual_seed(seed) torch.backends.cudnn.deterministic = True torch.backends.cudnn.benchmark = False def run_training(script_name, config_name, cudnn_benchmark=True, local_rank=-1, save_dir=None, base_seed=None, use_lmdb=False, script_name_prv=None, config_name_prv=None, use_wandb=False, distill=None, script_teacher=None, config_teacher=None): """Run the train script. args: script_name: Name of emperiment in the "experiments/" folder. config_name: Name of the yaml file in the "experiments/<script_name>". cudnn_benchmark: Use cudnn benchmark or not (default is True). """ if save_dir is None: print("save_dir dir is not given. Use the default dir instead.") # This is needed to avoid strange crashes related to opencv cv.setNumThreads(0) torch.backends.cudnn.benchmark = cudnn_benchmark print('script_name: {}.py config_name: {}.yaml'.format(script_name, config_name)) '''2021.1.5 set seed for different process''' if base_seed is not None: if local_rank != -1: init_seeds(base_seed + local_rank) else: init_seeds(base_seed) settings = ws_settings.Settings() settings.script_name = script_name settings.config_name = config_name settings.project_path = 'train/{}/{}'.format(script_name, config_name) if script_name_prv is not None and config_name_prv is not None: settings.project_path_prv = 'train/{}/{}'.format(script_name_prv, config_name_prv) settings.local_rank = local_rank settings.save_dir = os.path.abspath(save_dir) settings.use_lmdb = use_lmdb prj_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), "../..")) settings.cfg_file = os.path.join(prj_dir, 'experiments/%s/%s.yaml' % (script_name, config_name)) settings.use_wandb = use_wandb if distill: settings.distill = distill settings.script_teacher = script_teacher settings.config_teacher = config_teacher if script_teacher is not None and config_teacher is not None: settings.project_path_teacher = 'train/{}/{}'.format(script_teacher, config_teacher) settings.cfg_file_teacher = os.path.join(prj_dir, 'experiments/%s/%s.yaml' % (script_teacher, config_teacher)) expr_module = importlib.import_module('lib.train.train_script_distill') else: expr_module = importlib.import_module('lib.train.train_script') expr_func = getattr(expr_module, 'run') expr_func(settings) def main(): parser = argparse.ArgumentParser(description='Run a train scripts in train_settings.') parser.add_argument('--script', type=str, required=True, help='Name of the train script.') parser.add_argument('--config', type=str, required=True, help="Name of the config file.") parser.add_argument('--cudnn_benchmark', type=bool, default=True, help='Set cudnn benchmark on (1) or off (0) (default is on).') parser.add_argument('--local_rank', default=-1, type=int, help='node rank for distributed training') parser.add_argument('--save_dir', type=str, help='the directory to save checkpoints and logs') parser.add_argument('--seed', type=int, default=3407, help='seed for random numbers, default=42') parser.add_argument('--use_lmdb', type=int, choices=[0, 1], default=0) # whether datasets are in lmdb format parser.add_argument('--script_prv', type=str, default=None, help='Name of the train script of previous model.') parser.add_argument('--config_prv', type=str, default=None, help="Name of the config file of previous model.") parser.add_argument('--use_wandb', type=int, choices=[0, 1], default=0) # whether to use wandb # for knowledge distillation parser.add_argument('--distill', type=int, choices=[0, 1], default=0) # whether to use knowledge distillation parser.add_argument('--script_teacher', type=str, help='teacher script name') parser.add_argument('--config_teacher', type=str, help='teacher yaml configure file name') args = parser.parse_args() # 读取 local_rank 环境变量 if 'LOCAL_RANK' in os.environ: args.local_rank = int(os.environ['LOCAL_RANK']) # 设置 CUDA 设备 if args.local_rank != -1: dist.init_process_group(backend='nccl') torch.cuda.set_device(args.local_rank) else: torch.cuda.set_device(0) run_training(args.script, args.config, cudnn_benchmark=args.cudnn_benchmark, local_rank=args.local_rank, save_dir=args.save_dir, base_seed=args.seed, use_lmdb=args.use_lmdb, script_name_prv=args.script_prv, config_name_prv=args.config_prv, use_wandb=args.use_wandb, distill=args.distill, script_teacher=args.script_teacher, config_teacher=args.config_teacher) if __name__ == '__main__': main()这两段改动将确保每个进程设置自己的 GPU 设备,使其能够正确地使用多张 GPU 进行训练。你可以按照新的方式执行命令以启动多 GPU 训练:
torchrun --nproc_per_node=2 tracking/train.py --script ceutrack --config ceutrack_coesot --save_dir ./output --use_wandb 0这样,两个进程将分别使用不同的 GPU。
- User
不行,还是至使用gpu0号显卡
- Assistant
如果还是只使用0号显卡,问题可能在于环境变量的配置或者在代码中的设备设置逻辑不正确。为了更清楚地进行调试,我们可以采用更明确的设备分配方法。
更改方案
在多GPU环境下,使用
torch.distributed库自动处理设备分配和通信。在实际应用中,确保每个进程绑定到不同的GPU设备上十分重要。我们可以尝试使用torch.cuda.set_device来显式地设置GPU设备。确保你已经安装了
torch.distributed相关的包:pip install torch改动代码
将以下两处加入代码确保每个进程分配到一个特定的GPU:
Step 1: 读取local_rank环境变量
在解析命令行参数之后添加:
if 'LOCAL_RANK' in os.environ: args.local_rank = int(os.environ['LOCAL_RANK']) else: args.local_rank = 0Step 2: 配置CUDA设备和初始化进程组
在main函数初始化设备时加入:
if args.local_rank != -1: torch.cuda.set_device(args.local_rank) dist.init_process_group(backend='nccl') else: torch.cuda.set_device(0)修改后的代码
完整代码如下:
import os import sys import argparse import importlib import cv2 as cv import torch.backends.cudnn import torch.distributed as dist import torch import random import numpy as np torch.backends.cudnn.benchmark = False import _init_paths import lib.train.admin.settings as ws_settings def init_seeds(seed): random.seed(seed) np.random.seed(seed) torch.manual_seed(seed) torch.cuda.manual_seed(seed) torch.backends.cudnn.deterministic = True torch.backends.cudnn.benchmark = False def run_training(script_name, config_name, cudnn_benchmark=True, local_rank=-1, save_dir=None, base_seed=None, use_lmdb=False, script_name_prv=None, config_name_prv=None, use_wandb=False, distill=None, script_teacher=None, config_teacher=None): """Run the train script. args: script_name: Name of emperiment in the "experiments/" folder. config_name: Name of the yaml file in the "experiments/<script_name>". cudnn_benchmark: Use cudnn benchmark or not (default is True). """ if save_dir is None: print("save_dir dir is not given. Use the default dir instead.") # This is needed to avoid strange crashes related to opencv cv.setNumThreads(0) torch.backends.cudnn.benchmark = cudnn_benchmark print('script_name: {}.py config_name: {}.yaml'.format(script_name, config_name)) '''2021.1.5 set seed for different process''' if base_seed is not None: if local_rank != -1: init_seeds(base_seed + local_rank) else: init_seeds(base_seed) settings = ws_settings.Settings() settings.script_name = script_name settings.config_name = config_name settings.project_path = 'train/{}/{}'.format(script_name, config_name) if script_name_prv is not None and config_name_prv is not None: settings.project_path_prv = 'train/{}/{}'.format(script_name_prv, config_name_prv) settings.local_rank = local_rank settings.save_dir = os.path.abspath(save_dir) settings.use_lmdb = use_lmdb prj_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), "../..")) settings.cfg_file = os.path.join(prj_dir, 'experiments/%s/%s.yaml' % (script_name, config_name)) settings.use_wandb = use_wandb if distill: settings.distill = distill settings.script_teacher = script_teacher settings.config_teacher = config_teacher if script_teacher is not None and config_teacher is not None: settings.project_path_teacher = 'train/{}/{}'.format(script_teacher, config_teacher) settings.cfg_file_teacher = os.path.join(prj_dir, 'experiments/%s/%s.yaml' % (script_teacher, config_teacher)) expr_module = importlib.import_module('lib.train.train_script_distill') else: expr_module = importlib.import_module('lib.train.train_script') expr_func = getattr(expr_module, 'run') expr_func(settings) def main(): parser = argparse.ArgumentParser(description='Run a train scripts in train_settings.') parser.add_argument('--script', type=str, required=True, help='Name of the train script.') parser.add_argument('--config', type=str, required=True, help="Name of the config file.") parser.add_argument('--cudnn_benchmark', type=bool, default=True, help='Set cudnn benchmark on (1) or off (0) (default is on).') parser.add_argument('--local_rank', default=-1, type=int, help='node rank for distributed training') parser.add_argument('--save_dir', type=str, help='the directory to save checkpoints and logs') parser.add_argument('--seed', type=int, default=3407, help='seed for random numbers, default=42') parser.add_argument('--use_lmdb', type=int, choices=[0, 1], default=0, help="Whether datasets are in lmdb format") parser.add_argument('--script_prv', type=str, default=None, help='Name of the train script of previous model.') parser.add_argument('--config_prv', type=str, default=None, help="Name of the config file of previous model.") parser.add_argument('--use_wandb', type=int, choices=[0, 1], default=0, help="Whether to use wandb") # for knowledge distillation parser.add_argument('--distill', type=int, choices=[0, 1], default=0, help="Whether to use knowledge distillation") parser.add_argument('--script_teacher', type=str, help='Teacher script name') parser.add_argument('--config_teacher', type=str, help='Teacher yaml configure file name') args = parser.parse_args() # 读取local_rank环境变量 if 'LOCAL_RANK' in os.environ: args.local_rank = int(os.environ['LOCAL_RANK']) else: args.local_rank = 0 # 设置 CUDA 设备 if args.local_rank != -1: torch.cuda.set_device(args.local_rank) dist.init_process_group(backend='nccl') else: torch.cuda.set_device(0) run_training(args.script, args.config, cudnn_benchmark=args.cudnn_benchmark, local_rank=args.local_rank, save_dir=args.save_dir, base_seed=args.seed, use_lmdb=args.use_lmdb, script_name_prv=args.script_prv, config_name_prv=args.config_prv, use_wandb=args.use_wandb, distill=args.distill, script_teacher=args.script_teacher, config_teacher=args.config_teacher) if __name__ == '__main__': main()这样,你的代码显式地读取了
LOCAL_RANK环境变量并根据其值设置了对应的CUDA设备,然后初始化过程组。确保运行时使用的每个进程绑定到给定的GPU,这样你就能利用多GPU环境进行了。执行命令应如下:torchrun --nproc_per_node=2 tracking/train.py --script ceutrack --config ceutrack_coesot --save_dir ./output --use_wandb 0