在import os import sys import argparse import importlib import cv2 as cv import…
- User
在import os import sys import argparse import importlib import cv2 as cv import torch.backends.cudnn import torch.distributed as dist
import random import numpy as np torch.backends.cudnn.benchmark = False
import _init_paths import lib.train.admin.settings as ws_settings
def init_seeds(seed): random.seed(seed) np.random.seed(seed) torch.manual_seed(seed) torch.cuda.manual_seed(seed) torch.backends.cudnn.deterministic = True torch.backends.cudnn.benchmark = False
def run_training(script_name, config_name, cudnn_benchmark=True, local_rank=-1, save_dir=None, base_seed=None, use_lmdb=False, script_name_prv=None, config_name_prv=None, use_wandb=False, distill=None, script_teacher=None, config_teacher=None): """Run the train script. args: script_name: Name of emperiment in the "experiments/" folder. config_name: Name of the yaml file in the "experiments/<script_name>". cudnn_benchmark: Use cudnn benchmark or not (default is True). """ if save_dir is None: print("save_dir dir is not given. Use the default dir instead.") # This is needed to avoid strange crashes related to opencv cv.setNumThreads(0)
torch.backends.cudnn.benchmark = cudnn_benchmark print('script_name: {}.py config_name: {}.yaml'.format(script_name, config_name)) '''2021.1.5 set seed for different process''' if base_seed is not None: if local_rank != -1: init_seeds(base_seed + local_rank) else: init_seeds(base_seed) settings = ws_settings.Settings() settings.script_name = script_name settings.config_name = config_name settings.project_path = 'train/{}/{}'.format(script_name, config_name) if script_name_prv is not None and config_name_prv is not None: settings.project_path_prv = 'train/{}/{}'.format(script_name_prv, config_name_prv) settings.local_rank = local_rank settings.save_dir = os.path.abspath(save_dir) settings.use_lmdb = use_lmdb prj_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), "../..")) settings.cfg_file = os.path.join(prj_dir, 'experiments/%s/%s.yaml' % (script_name, config_name)) settings.use_wandb = use_wandb if distill: settings.distill = distill settings.script_teacher = script_teacher settings.config_teacher = config_teacher if script_teacher is not None and config_teacher is not None: settings.project_path_teacher = 'train/{}/{}'.format(script_teacher, config_teacher) settings.cfg_file_teacher = os.path.join(prj_dir, 'experiments/%s/%s.yaml' % (script_teacher, config_teacher)) expr_module = importlib.import_module('lib.train.train_script_distill') else: expr_module = importlib.import_module('lib.train.train_script') expr_func = getattr(expr_module, 'run') expr_func(settings)def main(): parser = argparse.ArgumentParser(description='Run a train scripts in train_settings.') parser.add_argument('--script', type=str, required=True, help='Name of the train script.') parser.add_argument('--config', type=str, required=True, help="Name of the config file.") parser.add_argument('--cudnn_benchmark', type=bool, default=True, help='Set cudnn benchmark on (1) or off (0) (default is on).') # parser.add_argument('--local_rank', default=-1, type=int, help='node rank for distributed training') parser.add_argument('--save_dir', type=str, help='the directory to save checkpoints and logs') parser.add_argument('--seed', type=int, default=3407, help='seed for random numbers, default=42') parser.add_argument('--use_lmdb', type=int, choices=[0, 1], default=0) # whether datasets are in lmdb format parser.add_argument('--script_prv', type=str, default=None, help='Name of the train script of previous model.') parser.add_argument('--config_prv', type=str, default=None, help="Name of the config file of previous model.") parser.add_argument('--use_wandb', type=int, choices=[0, 1], default=0) # whether to use wandb # for knowledge distillation parser.add_argument('--distill', type=int, choices=[0, 1], default=0) # whether to use knowledge distillation parser.add_argument('--script_teacher', type=str, help='teacher script name') parser.add_argument('--config_teacher', type=str, help='teacher yaml configure file name')
args = parser.parse_args() ############## if 'LOCAL_RANK' in os.environ: args.local_rank = int(os.environ['LOCAL_RANK']) #################### if args.local_rank != -1: dist.init_process_group(backend='nccl') torch.cuda.set_device(args.local_rank) else: torch.cuda.set_device(0) run_training(args.script, args.config, cudnn_benchmark=args.cudnn_benchmark, local_rank=args.local_rank, save_dir=args.save_dir, base_seed=args.seed, use_lmdb=args.use_lmdb, script_name_prv=args.script_prv, config_name_prv=args.config_prv, use_wandb=args.use_wandb, distill=args.distill, script_teacher=args.script_teacher, config_teacher=args.config_teacher)if name == 'main': # os.environ['CUDA_VISIBLE_DEVICES'] = "4" main() 和 import os import argparse import random
def parse_args(): """ args for training. """ parser = argparse.ArgumentParser(description='Parse args for training') # for train parser.add_argument('--script', type=str, help='training script name') parser.add_argument('--config', type=str, default='baseline', help='yaml configure file name') parser.add_argument('--save_dir', type=str, help='root directory to save checkpoints, logs, and tensorboard') parser.add_argument('--mode', type=str, choices=["single", "multiple", "multi_node"], default="multiple", help="train on single gpu or multiple gpus") # parser.add_argument('--nproc_per_node', type=int, help="number of GPUs per node") # specify when mode is multiple parser.add_argument('--nproc_per_node', type=int, default=1, help="number of GPUs per node") parser.add_argument('--use_lmdb', type=int, choices=[0, 1], default=0) # whether datasets are in lmdb format parser.add_argument('--script_prv', type=str, help='training script name') parser.add_argument('--config_prv', type=str, default='baseline', help='yaml configure file name') parser.add_argument('--use_wandb', type=int, choices=[0, 1], default=0) # whether to use wandb # for knowledge distillation parser.add_argument('--distill', type=int, choices=[0, 1], default=0) # whether to use knowledge distillation parser.add_argument('--script_teacher', type=str, help='teacher script name') parser.add_argument('--config_teacher', type=str, help='teacher yaml configure file name')
# for multiple machines parser.add_argument('--rank', type=int, help='Rank of the current process.') parser.add_argument('--world-size', type=int, help='Number of processes participating in the job.') parser.add_argument('--ip', type=str, default='127.0.0.1', help='IP of the current rank 0.') parser.add_argument('--port', type=int, default='20000', help='Port of the current rank 0.') args = parser.parse_args() return argsdef main(): args = parse_args() if args.mode == "single": train_cmd = "python lib/train/run_training.py --script %s --config %s --save_dir %s --use_lmdb %d "
"--script_prv %s --config_prv %s --distill %d --script_teacher %s --config_teacher %s --use_wandb %d"
% (args.script, args.config, args.save_dir, args.use_lmdb, args.script_prv, args.config_prv, args.distill, args.script_teacher, args.config_teacher, args.use_wandb) elif args.mode == "multiple": train_cmd = "torchrun --nproc_per_node %d --master_port %d lib/train/run_training.py "
"--script %s --config %s --save_dir %s --use_lmdb %d --script_prv %s --config_prv %s --use_wandb %d "
"--distill %d --script_teacher %s --config_teacher %s"
% (args.nproc_per_node, random.randint(10000, 50000), args.script, args.config, args.save_dir, args.use_lmdb, args.script_prv, args.config_prv, args.use_wandb, args.distill, args.script_teacher, args.config_teacher) elif args.mode == "multi_node": train_cmd = "python -m torch.distributed.launch --nproc_per_node %d --master_addr %s --master_port %d --nnodes %d --node_rank %d lib/train/run_training.py "
"--script %s --config %s --save_dir %s --use_lmdb %d --script_prv %s --config_prv %s --use_wandb %d "
"--distill %d --script_teacher %s --config_teacher %s"
% (args.nproc_per_node, args.ip, args.port, args.world_size, args.rank, args.script, args.config, args.save_dir, args.use_lmdb, args.script_prv, args.config_prv, args.use_wandb, args.distill, args.script_teacher, args.config_teacher) else: raise ValueError("mode should be 'single' or 'multiple'.") os.system(train_cmd)if name == "main": main() ,运行命令是:torchrun --nproc_per_node=2 tracking/train.py --script ceutrack --config ceutrack_coesot --save_dir ./output --use_wandb 0,但是目前在一块显卡上,分两步,我们期望在两块显卡上运行代码。
- Assistant
要在两块显卡上运行该代码,可以按照以下步骤进行:
-
确保你有两块可用的GPU: 使用命令
nvidia-smi来检查你的系统上是否有两块可用的GPU。 -
设置正确的环境变量: 确保
CUDA_VISIBLE_DEVICES环境变量已设置为两块显卡。在运行实际命令之前,你可以在命令行中设置环境变量:export CUDA_VISIBLE_DEVICES=0,1 -
运行命令: 使用
torchrun或python -m torch.distributed.launch来分布式运行代码。你已经选择了torchrun,所以你只需要调整配置,并确保一些必要的标志。
以下是推荐的设置和命令:
使用
torchruntorchrun是 PyTorch的新推荐工具,用于分布式启动。export CUDA_VISIBLE_DEVICES=0,1 torchrun --nproc_per_node=2 tracking/train.py --script ceutrack --config ceutrack_coesot --save_dir ./output --use_wandb 0说明:
export CUDA_VISIBLE_DEVICES=0,1:设置可见的 GPU。--nproc_per_node=2:告诉torchrun使用两块GPU。- 其余参数根据你的训练脚本所需指定。
检查代码中的分布式设置
确保代码中可以正确设置每块GPU。特别地,训练脚本
run_training需要正确初始化分布式环境。你的run_training函数已经包含了所需的逻辑:if args.local_rank != -1: dist.init_process_group(backend='nccl') torch.cuda.set_device(args.local_rank) else: torch.cuda.set_device(0)这个部分确保在多卡训练时,每个进程运行所在的独立显卡。
检查多进程启动
为确保每个进程都能正确启动,可以在你的代码中继续保留
torch.distributed的初始化逻辑。总结下,最关键的部分是确保显卡可见并且有正确设定的
torchrun启动命令。按照以上步骤执行,可以在两块显卡上运行训练代码。 -
- User
我们执行了export CUDA_VISIBLE_DEVICES=0,1 torchrun --nproc_per_node=2 tracking/train.py --script ceutrack --config ceutrack_coesot --save_dir ./output --use_wandb 0,现在观察显卡,就仅仅利用了0号显卡,然后执行了两步;1号显卡并未使用。
- Assistant
确保多GPU正确配置与使用
如果你已正确设置了
CUDA_VISIBLE_DEVICES环境变量,但发现仅0号显卡被使用,那么可能有以下几种原因:-
环境变量设置检查: 确认你在同一个终端中执行了
export CUDA_VISIBLE_DEVICES=0,1并随后运行了torchrun命令。 -
代码检查: 确保代码正确地分发了任务到每块GPU上。你在代码中的
local_rank检查对于多GPU是必要的。
调试步骤
1. 设置环境变量并验证
确保环境变量已正确设置,并输出验证:
export CUDA_VISIBLE_DEVICES=0,1 echo $CUDA_VISIBLE_DEVICES # Should output 0,12. 验证
torchrun命令确认以下命令是否按照设想执行:
torchrun --nproc_per_node=2 tracking/train.py --script ceutrack --config ceutrack_coesot --save_dir ./output --use_wandb 03. 检查代码中的
local_rank分配你的代码看起来处理了
local_rank的逻辑。只是为了双重确认,确保以下代码片段被正确执行:if 'LOCAL_RANK' in os.environ: args.local_rank = int(os.environ['LOCAL_RANK'])以及:
if args.local_rank != -1: dist.init_process_group(backend='nccl') torch.cuda.set_device(args.local_rank) else: torch.cuda.set_device(0)4. 代码中的日志信息
为了更好的调试,让我们在设置
CUDA设备时添加一些日志信息:if args.local_rank != -1: print(f"Initializing process group for local rank: {args.local_rank}") dist.init_process_group(backend='nccl') print(f"Setting CUDA device to local rank: {args.local_rank}") torch.cuda.set_device(args.local_rank) else: print("Setting CUDA device to 0") torch.cuda.set_device(0)通过这样的输出可以确保每个进程启动时,知道它在使用哪块显卡。
5. 利用
torch.cuda.device_count()在代码中加入以下行来检查设备数量:
print(f"Number of available CUDA devices: {torch.cuda.device_count()}")如果
device_count不对,那可能是环境变量或者 PyTorch 安装问题。完整可能的
main()函数作为参考,以下是修改后的
main()函数:def main(): parser = argparse.ArgumentParser(description='Run a train scripts in train_settings.') parser.add_argument('--script', type=str, required=True, help='Name of the train script.') parser.add_argument('--config', type=str, required=True, help="Name of the config file.") parser.add_argument('--cudnn_benchmark', type=bool, default=True, help='Set cudnn benchmark on (1) or off (0) (default is on).') parser.add_argument('--save_dir', type=str, help='the directory to save checkpoints and logs') parser.add_argument('--seed', type=int, default=3407, help='seed for random numbers, default=42') parser.add_argument('--use_lmdb', type=int, choices=[0, 1], default=0) parser.add_argument('--script_prv', type=str, default=None, help='Name of the train script of previous model.') parser.add_argument('--config_prv', type=str, default=None, help="Name of the config file of previous model.") parser.add_argument('--use_wandb', type=int, choices=[0, 1], default=0) parser.add_argument('--distill', type=int, choices=[0, 1], default=0) parser.add_argument('--script_teacher', type=str, help='teacher script name') parser.add_argument('--config_teacher', type=str, help='teacher yaml configure file name') args = parser.parse_args() if 'LOCAL_RANK' in os.environ: args.local_rank = int(os.environ['LOCAL_RANK']) else: args.local_rank = -1 # default should be -1 if LOCAL_RANK is not set if args.local_rank != -1: print(f"Initializing process group for local rank: {args.local_rank}") dist.init_process_group(backend='nccl') print(f"Setting CUDA device to local rank: {args.local_rank}") torch.cuda.set_device(args.local_rank) else: print("Setting CUDA device to 0") torch.cuda.set_device(0) print(f"Number of available CUDA devices: {torch.cuda.device_count()}") run_training(args.script, args.config, cudnn_benchmark=args.cudnn_benchmark, local_rank=args.local_rank, save_dir=args.save_dir, base_seed=args.seed, use_lmdb=args.use_lmdb, script_name_prv=args.script_prv, config_name_prv=args.config_prv, use_wandb=args.use_wandb, distill=args.distill, script_teacher=args.script_teacher, config_teacher=args.config_teacher)通过这些检查和调试信息,你应该能够更好地确定是什么阻止了多GPU的使用。
-