Skip to content
TheChatArchiveThe Chat Archive
gpt-4o-2024-05-13· OpenAIAi2 WildChat-4.8M (AI2)

在import os import sys import argparse import importlib import cv2 as cv import…

  1. User

    在import os import sys import argparse import importlib import cv2 as cv import torch.backends.cudnn import torch.distributed as dist

    import random import numpy as np torch.backends.cudnn.benchmark = False

    import _init_paths import lib.train.admin.settings as ws_settings

    def init_seeds(seed): random.seed(seed) np.random.seed(seed) torch.manual_seed(seed) torch.cuda.manual_seed(seed) torch.backends.cudnn.deterministic = True torch.backends.cudnn.benchmark = False

    def run_training(script_name, config_name, cudnn_benchmark=True, local_rank=-1, save_dir=None, base_seed=None, use_lmdb=False, script_name_prv=None, config_name_prv=None, use_wandb=False, distill=None, script_teacher=None, config_teacher=None): """Run the train script. args: script_name: Name of emperiment in the "experiments/" folder. config_name: Name of the yaml file in the "experiments/<script_name>". cudnn_benchmark: Use cudnn benchmark or not (default is True). """ if save_dir is None: print("save_dir dir is not given. Use the default dir instead.") # This is needed to avoid strange crashes related to opencv cv.setNumThreads(0)

    torch.backends.cudnn.benchmark = cudnn_benchmark
    
    print('script_name: {}.py  config_name: {}.yaml'.format(script_name, config_name))
    
    '''2021.1.5 set seed for different process'''
    if base_seed is not None:
        if local_rank != -1:
            init_seeds(base_seed + local_rank)
        else:
            init_seeds(base_seed)
    
    settings = ws_settings.Settings()
    settings.script_name = script_name
    settings.config_name = config_name
    settings.project_path = 'train/{}/{}'.format(script_name, config_name)
    if script_name_prv is not None and config_name_prv is not None:
        settings.project_path_prv = 'train/{}/{}'.format(script_name_prv, config_name_prv)
    settings.local_rank = local_rank
    settings.save_dir = os.path.abspath(save_dir)
    settings.use_lmdb = use_lmdb
    prj_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), "../.."))
    settings.cfg_file = os.path.join(prj_dir, 'experiments/%s/%s.yaml' % (script_name, config_name))
    settings.use_wandb = use_wandb
    if distill:
        settings.distill = distill
        settings.script_teacher = script_teacher
        settings.config_teacher = config_teacher
        if script_teacher is not None and config_teacher is not None:
            settings.project_path_teacher = 'train/{}/{}'.format(script_teacher, config_teacher)
        settings.cfg_file_teacher = os.path.join(prj_dir, 'experiments/%s/%s.yaml' % (script_teacher, config_teacher))
        expr_module = importlib.import_module('lib.train.train_script_distill')
    else:
        expr_module = importlib.import_module('lib.train.train_script')
    expr_func = getattr(expr_module, 'run')
    
    expr_func(settings)
    

    def main(): parser = argparse.ArgumentParser(description='Run a train scripts in train_settings.') parser.add_argument('--script', type=str, required=True, help='Name of the train script.') parser.add_argument('--config', type=str, required=True, help="Name of the config file.") parser.add_argument('--cudnn_benchmark', type=bool, default=True, help='Set cudnn benchmark on (1) or off (0) (default is on).') # parser.add_argument('--local_rank', default=-1, type=int, help='node rank for distributed training') parser.add_argument('--save_dir', type=str, help='the directory to save checkpoints and logs') parser.add_argument('--seed', type=int, default=3407, help='seed for random numbers, default=42') parser.add_argument('--use_lmdb', type=int, choices=[0, 1], default=0) # whether datasets are in lmdb format parser.add_argument('--script_prv', type=str, default=None, help='Name of the train script of previous model.') parser.add_argument('--config_prv', type=str, default=None, help="Name of the config file of previous model.") parser.add_argument('--use_wandb', type=int, choices=[0, 1], default=0) # whether to use wandb # for knowledge distillation parser.add_argument('--distill', type=int, choices=[0, 1], default=0) # whether to use knowledge distillation parser.add_argument('--script_teacher', type=str, help='teacher script name') parser.add_argument('--config_teacher', type=str, help='teacher yaml configure file name')

    args = parser.parse_args()
    ##############
    if 'LOCAL_RANK' in os.environ:
        args.local_rank = int(os.environ['LOCAL_RANK'])
    ####################
    if args.local_rank != -1:
        dist.init_process_group(backend='nccl')
        torch.cuda.set_device(args.local_rank)
    else:
        torch.cuda.set_device(0)
    run_training(args.script, args.config, cudnn_benchmark=args.cudnn_benchmark,
                 local_rank=args.local_rank, save_dir=args.save_dir, base_seed=args.seed,
                 use_lmdb=args.use_lmdb, script_name_prv=args.script_prv, config_name_prv=args.config_prv,
                 use_wandb=args.use_wandb,
                 distill=args.distill, script_teacher=args.script_teacher, config_teacher=args.config_teacher)
    

    if name == 'main': # os.environ['CUDA_VISIBLE_DEVICES'] = "4" main() 和 import os import argparse import random

    def parse_args(): """ args for training. """ parser = argparse.ArgumentParser(description='Parse args for training') # for train parser.add_argument('--script', type=str, help='training script name') parser.add_argument('--config', type=str, default='baseline', help='yaml configure file name') parser.add_argument('--save_dir', type=str, help='root directory to save checkpoints, logs, and tensorboard') parser.add_argument('--mode', type=str, choices=["single", "multiple", "multi_node"], default="multiple", help="train on single gpu or multiple gpus") # parser.add_argument('--nproc_per_node', type=int, help="number of GPUs per node") # specify when mode is multiple parser.add_argument('--nproc_per_node', type=int, default=1, help="number of GPUs per node") parser.add_argument('--use_lmdb', type=int, choices=[0, 1], default=0) # whether datasets are in lmdb format parser.add_argument('--script_prv', type=str, help='training script name') parser.add_argument('--config_prv', type=str, default='baseline', help='yaml configure file name') parser.add_argument('--use_wandb', type=int, choices=[0, 1], default=0) # whether to use wandb # for knowledge distillation parser.add_argument('--distill', type=int, choices=[0, 1], default=0) # whether to use knowledge distillation parser.add_argument('--script_teacher', type=str, help='teacher script name') parser.add_argument('--config_teacher', type=str, help='teacher yaml configure file name')

    # for multiple machines
    parser.add_argument('--rank', type=int, help='Rank of the current process.')
    parser.add_argument('--world-size', type=int, help='Number of processes participating in the job.')
    parser.add_argument('--ip', type=str, default='127.0.0.1', help='IP of the current rank 0.')
    parser.add_argument('--port', type=int, default='20000', help='Port of the current rank 0.')
    
    args = parser.parse_args()
    
    return args
    

    def main(): args = parse_args() if args.mode == "single": train_cmd = "python lib/train/run_training.py --script %s --config %s --save_dir %s --use_lmdb %d "
    "--script_prv %s --config_prv %s --distill %d --script_teacher %s --config_teacher %s --use_wandb %d"
    % (args.script, args.config, args.save_dir, args.use_lmdb, args.script_prv, args.config_prv, args.distill, args.script_teacher, args.config_teacher, args.use_wandb) elif args.mode == "multiple": train_cmd = "torchrun --nproc_per_node %d --master_port %d lib/train/run_training.py "
    "--script %s --config %s --save_dir %s --use_lmdb %d --script_prv %s --config_prv %s --use_wandb %d "
    "--distill %d --script_teacher %s --config_teacher %s"
    % (args.nproc_per_node, random.randint(10000, 50000), args.script, args.config, args.save_dir, args.use_lmdb, args.script_prv, args.config_prv, args.use_wandb, args.distill, args.script_teacher, args.config_teacher) elif args.mode == "multi_node": train_cmd = "python -m torch.distributed.launch --nproc_per_node %d --master_addr %s --master_port %d --nnodes %d --node_rank %d lib/train/run_training.py "
    "--script %s --config %s --save_dir %s --use_lmdb %d --script_prv %s --config_prv %s --use_wandb %d "
    "--distill %d --script_teacher %s --config_teacher %s"
    % (args.nproc_per_node, args.ip, args.port, args.world_size, args.rank, args.script, args.config, args.save_dir, args.use_lmdb, args.script_prv, args.config_prv, args.use_wandb, args.distill, args.script_teacher, args.config_teacher) else: raise ValueError("mode should be 'single' or 'multiple'.") os.system(train_cmd)

    if name == "main": main() ,运行命令是:torchrun --nproc_per_node=2 tracking/train.py --script ceutrack --config ceutrack_coesot --save_dir ./output --use_wandb 0,但是目前在一块显卡上,分两步,我们期望在两块显卡上运行代码。

  2. Assistant

    要在两块显卡上运行该代码,可以按照以下步骤进行:

    1. 确保你有两块可用的GPU: 使用命令 nvidia-smi 来检查你的系统上是否有两块可用的GPU。

    2. 设置正确的环境变量: 确保 CUDA_VISIBLE_DEVICES 环境变量已设置为两块显卡。在运行实际命令之前,你可以在命令行中设置环境变量:

      export CUDA_VISIBLE_DEVICES=0,1
      
    3. 运行命令: 使用 torchrun 或 python -m torch.distributed.launch 来分布式运行代码。你已经选择了 torchrun,所以你只需要调整配置,并确保一些必要的标志。

    以下是推荐的设置和命令:

    使用 torchrun

    torchrun 是 PyTorch的新推荐工具,用于分布式启动。

    export CUDA_VISIBLE_DEVICES=0,1
    torchrun --nproc_per_node=2 tracking/train.py --script ceutrack --config ceutrack_coesot --save_dir ./output --use_wandb 0
    

    说明:

    • export CUDA_VISIBLE_DEVICES=0,1:设置可见的 GPU。
    • --nproc_per_node=2:告诉 torchrun 使用两块GPU。
    • 其余参数根据你的训练脚本所需指定。

    检查代码中的分布式设置

    确保代码中可以正确设置每块GPU。特别地,训练脚本 run_training 需要正确初始化分布式环境。你的 run_training 函数已经包含了所需的逻辑:

    if args.local_rank != -1:
        dist.init_process_group(backend='nccl')
        torch.cuda.set_device(args.local_rank)
    else:
        torch.cuda.set_device(0)
    

    这个部分确保在多卡训练时,每个进程运行所在的独立显卡。

    检查多进程启动

    为确保每个进程都能正确启动,可以在你的代码中继续保留 torch.distributed 的初始化逻辑。

    总结下,最关键的部分是确保显卡可见并且有正确设定的torchrun启动命令。按照以上步骤执行,可以在两块显卡上运行训练代码。

  3. User

    我们执行了export CUDA_VISIBLE_DEVICES=0,1 torchrun --nproc_per_node=2 tracking/train.py --script ceutrack --config ceutrack_coesot --save_dir ./output --use_wandb 0,现在观察显卡,就仅仅利用了0号显卡,然后执行了两步;1号显卡并未使用。

  4. Assistant

    确保多GPU正确配置与使用

    如果你已正确设置了 CUDA_VISIBLE_DEVICES 环境变量,但发现仅0号显卡被使用,那么可能有以下几种原因:

    1. 环境变量设置检查: 确认你在同一个终端中执行了 export CUDA_VISIBLE_DEVICES=0,1 并随后运行了 torchrun 命令。

    2. 代码检查: 确保代码正确地分发了任务到每块GPU上。你在代码中的 local_rank 检查对于多GPU是必要的。

    调试步骤

    1. 设置环境变量并验证

    确保环境变量已正确设置,并输出验证:

    export CUDA_VISIBLE_DEVICES=0,1
    echo $CUDA_VISIBLE_DEVICES  # Should output 0,1
    

    2. 验证 torchrun 命令

    确认以下命令是否按照设想执行:

    torchrun --nproc_per_node=2 tracking/train.py --script ceutrack --config ceutrack_coesot --save_dir ./output --use_wandb 0
    

    3. 检查代码中的 local_rank 分配

    你的代码看起来处理了 local_rank 的逻辑。只是为了双重确认,确保以下代码片段被正确执行:

    if 'LOCAL_RANK' in os.environ:
        args.local_rank = int(os.environ['LOCAL_RANK'])
    

    以及:

    if args.local_rank != -1:
        dist.init_process_group(backend='nccl')
        torch.cuda.set_device(args.local_rank)
    else:
        torch.cuda.set_device(0)
    

    4. 代码中的日志信息

    为了更好的调试,让我们在设置 CUDA 设备时添加一些日志信息:

    if args.local_rank != -1:
        print(f"Initializing process group for local rank: {args.local_rank}")
        dist.init_process_group(backend='nccl')
        print(f"Setting CUDA device to local rank: {args.local_rank}")
        torch.cuda.set_device(args.local_rank)
    else:
        print("Setting CUDA device to 0")
        torch.cuda.set_device(0)
    

    通过这样的输出可以确保每个进程启动时,知道它在使用哪块显卡。

    5. 利用 torch.cuda.device_count()

    在代码中加入以下行来检查设备数量:

    print(f"Number of available CUDA devices: {torch.cuda.device_count()}")
    

    如果 device_count 不对,那可能是环境变量或者 PyTorch 安装问题。

    完整可能的 main() 函数

    作为参考,以下是修改后的 main() 函数:

    def main():
        parser = argparse.ArgumentParser(description='Run a train scripts in train_settings.')
        parser.add_argument('--script', type=str, required=True, help='Name of the train script.')
        parser.add_argument('--config', type=str, required=True, help="Name of the config file.")
        parser.add_argument('--cudnn_benchmark', type=bool, default=True, help='Set cudnn benchmark on (1) or off (0) (default is on).')
        parser.add_argument('--save_dir', type=str, help='the directory to save checkpoints and logs')
        parser.add_argument('--seed', type=int, default=3407, help='seed for random numbers, default=42')
        parser.add_argument('--use_lmdb', type=int, choices=[0, 1], default=0)
        parser.add_argument('--script_prv', type=str, default=None, help='Name of the train script of previous model.')
        parser.add_argument('--config_prv', type=str, default=None, help="Name of the config file of previous model.")
        parser.add_argument('--use_wandb', type=int, choices=[0, 1], default=0)
        parser.add_argument('--distill', type=int, choices=[0, 1], default=0)
        parser.add_argument('--script_teacher', type=str, help='teacher script name')
        parser.add_argument('--config_teacher', type=str, help='teacher yaml configure file name')
    
        args = parser.parse_args()
        if 'LOCAL_RANK' in os.environ:
            args.local_rank = int(os.environ['LOCAL_RANK'])
        else:
            args.local_rank = -1  # default should be -1 if LOCAL_RANK is not set
    
        if args.local_rank != -1:
            print(f"Initializing process group for local rank: {args.local_rank}")
            dist.init_process_group(backend='nccl')
            print(f"Setting CUDA device to local rank: {args.local_rank}")
            torch.cuda.set_device(args.local_rank)
        else:
            print("Setting CUDA device to 0")
            torch.cuda.set_device(0)
        
        print(f"Number of available CUDA devices: {torch.cuda.device_count()}")
    
        run_training(args.script, args.config, cudnn_benchmark=args.cudnn_benchmark,
                     local_rank=args.local_rank, save_dir=args.save_dir, base_seed=args.seed,
                     use_lmdb=args.use_lmdb, script_name_prv=args.script_prv, config_name_prv=args.config_prv,
                     use_wandb=args.use_wandb,
                     distill=args.distill, script_teacher=args.script_teacher, config_teacher=args.config_teacher)
    

    通过这些检查和调试信息,你应该能够更好地确定是什么阻止了多GPU的使用。