Skip to content
TheChatArchiveThe Chat Archive
gpt-4o-2024-05-13· OpenAIAi2 WildChat-4.8M (AI2)

训练代码是:① import os import argparse import random def parseargs(): """ args for training.…

  1. User

    训练代码是:① import os import argparse import random

    def parse_args(): """ args for training. """ parser = argparse.ArgumentParser(description='Parse args for training') # for train parser.add_argument('--script', type=str, help='training script name') parser.add_argument('--config', type=str, default='baseline', help='yaml configure file name') parser.add_argument('--save_dir', type=str, help='root directory to save checkpoints, logs, and tensorboard') parser.add_argument('--mode', type=str, choices=["single", "multiple", "multi_node"], default="multiple", help="train on single gpu or multiple gpus") # parser.add_argument('--nproc_per_node', type=int, help="number of GPUs per node") # specify when mode is multiple parser.add_argument('--nproc_per_node', type=int, default=1, help="number of GPUs per node") parser.add_argument('--use_lmdb', type=int, choices=[0, 1], default=0) # whether datasets are in lmdb format parser.add_argument('--script_prv', type=str, help='training script name') parser.add_argument('--config_prv', type=str, default='baseline', help='yaml configure file name') parser.add_argument('--use_wandb', type=int, choices=[0, 1], default=0) # whether to use wandb # for knowledge distillation parser.add_argument('--distill', type=int, choices=[0, 1], default=0) # whether to use knowledge distillation parser.add_argument('--script_teacher', type=str, help='teacher script name') parser.add_argument('--config_teacher', type=str, help='teacher yaml configure file name')

    # for multiple machines
    parser.add_argument('--rank', type=int, help='Rank of the current process.')
    parser.add_argument('--world-size', type=int, help='Number of processes participating in the job.')
    parser.add_argument('--ip', type=str, default='127.0.0.1', help='IP of the current rank 0.')
    parser.add_argument('--port', type=int, default='20000', help='Port of the current rank 0.')
    
    args = parser.parse_args()
    
    return args
    

    def main(): args = parse_args() if args.mode == "single": train_cmd = "python lib/train/run_training.py --script %s --config %s --save_dir %s --use_lmdb %d "
    "--script_prv %s --config_prv %s --distill %d --script_teacher %s --config_teacher %s --use_wandb %d"
    % (args.script, args.config, args.save_dir, args.use_lmdb, args.script_prv, args.config_prv, args.distill, args.script_teacher, args.config_teacher, args.use_wandb) elif args.mode == "multiple": train_cmd = "torchrun --nproc_per_node %d --master_port %d lib/train/run_training.py "
    "--script %s --config %s --save_dir %s --use_lmdb %d --script_prv %s --config_prv %s --use_wandb %d "
    "--distill %d --script_teacher %s --config_teacher %s"
    % (args.nproc_per_node, random.randint(10000, 50000), args.script, args.config, args.save_dir, args.use_lmdb, args.script_prv, args.config_prv, args.use_wandb, args.distill, args.script_teacher, args.config_teacher) elif args.mode == "multi_node": train_cmd = "python -m torch.distributed.launch --nproc_per_node %d --master_addr %s --master_port %d --nnodes %d --node_rank %d lib/train/run_training.py "
    "--script %s --config %s --save_dir %s --use_lmdb %d --script_prv %s --config_prv %s --use_wandb %d "
    "--distill %d --script_teacher %s --config_teacher %s"
    % (args.nproc_per_node, args.ip, args.port, args.world_size, args.rank, args.script, args.config, args.save_dir, args.use_lmdb, args.script_prv, args.config_prv, args.use_wandb, args.distill, args.script_teacher, args.config_teacher) else: raise ValueError("mode should be 'single' or 'multiple'.") os.system(train_cmd)

    if name == "main": main() 和 ② import os import sys import argparse import importlib import cv2 as cv import torch.backends.cudnn import torch.distributed as dist

    import random import numpy as np torch.backends.cudnn.benchmark = False

    import _init_paths import lib.train.admin.settings as ws_settings

    def init_seeds(seed): random.seed(seed) np.random.seed(seed) torch.manual_seed(seed) torch.cuda.manual_seed(seed) torch.backends.cudnn.deterministic = True torch.backends.cudnn.benchmark = False

    def run_training(script_name, config_name, cudnn_benchmark=True, local_rank=-1, save_dir=None, base_seed=None, use_lmdb=False, script_name_prv=None, config_name_prv=None, use_wandb=False, distill=None, script_teacher=None, config_teacher=None): """Run the train script. args: script_name: Name of emperiment in the "experiments/" folder. config_name: Name of the yaml file in the "experiments/<script_name>". cudnn_benchmark: Use cudnn benchmark or not (default is True). """ if save_dir is None: print("save_dir dir is not given. Use the default dir instead.") # This is needed to avoid strange crashes related to opencv cv.setNumThreads(0)

    torch.backends.cudnn.benchmark = cudnn_benchmark
    
    print('script_name: {}.py  config_name: {}.yaml'.format(script_name, config_name))
    
    '''2021.1.5 set seed for different process'''
    if base_seed is not None:
        if local_rank != -1:
            init_seeds(base_seed + local_rank)
        else:
            init_seeds(base_seed)
    
    settings = ws_settings.Settings()
    settings.script_name = script_name
    settings.config_name = config_name
    settings.project_path = 'train/{}/{}'.format(script_name, config_name)
    if script_name_prv is not None and config_name_prv is not None:
        settings.project_path_prv = 'train/{}/{}'.format(script_name_prv, config_name_prv)
    settings.local_rank = local_rank
    settings.save_dir = os.path.abspath(save_dir)
    settings.use_lmdb = use_lmdb
    prj_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), "../.."))
    settings.cfg_file = os.path.join(prj_dir, 'experiments/%s/%s.yaml' % (script_name, config_name))
    settings.use_wandb = use_wandb
    if distill:
        settings.distill = distill
        settings.script_teacher = script_teacher
        settings.config_teacher = config_teacher
        if script_teacher is not None and config_teacher is not None:
            settings.project_path_teacher = 'train/{}/{}'.format(script_teacher, config_teacher)
        settings.cfg_file_teacher = os.path.join(prj_dir, 'experiments/%s/%s.yaml' % (script_teacher, config_teacher))
        expr_module = importlib.import_module('lib.train.train_script_distill')
    else:
        expr_module = importlib.import_module('lib.train.train_script')
    expr_func = getattr(expr_module, 'run')
    
    expr_func(settings)
    

    def main(): parser = argparse.ArgumentParser(description='Run a train scripts in train_settings.') parser.add_argument('--script', type=str, required=True, help='Name of the train script.') parser.add_argument('--config', type=str, required=True, help="Name of the config file.") parser.add_argument('--cudnn_benchmark', type=bool, default=True, help='Set cudnn benchmark on (1) or off (0) (default is on).') parser.add_argument('--local_rank', default=-1, type=int, help='node rank for distributed training') parser.add_argument('--save_dir', type=str, help='the directory to save checkpoints and logs') parser.add_argument('--seed', type=int, default=3407, help='seed for random numbers, default=42') parser.add_argument('--use_lmdb', type=int, choices=[0, 1], default=0) # whether datasets are in lmdb format parser.add_argument('--script_prv', type=str, default=None, help='Name of the train script of previous model.') parser.add_argument('--config_prv', type=str, default=None, help="Name of the config file of previous model.") parser.add_argument('--use_wandb', type=int, choices=[0, 1], default=0) # whether to use wandb # for knowledge distillation parser.add_argument('--distill', type=int, choices=[0, 1], default=0) # whether to use knowledge distillation parser.add_argument('--script_teacher', type=str, help='teacher script name') parser.add_argument('--config_teacher', type=str, help='teacher yaml configure file name')

    args = parser.parse_args()
    ##############
    if 'LOCAL_RANK' in os.environ:
        args.local_rank = int(os.environ['LOCAL_RANK'])
    ####################
    if args.local_rank != -1:
        dist.init_process_group(backend='nccl')
        torch.cuda.set_device(args.local_rank)
    else:
        torch.cuda.set_device(0)
    run_training(args.script, args.config, cudnn_benchmark=args.cudnn_benchmark,
                 local_rank=args.local_rank, save_dir=args.save_dir, base_seed=args.seed,
                 use_lmdb=args.use_lmdb, script_name_prv=args.script_prv, config_name_prv=args.config_prv,
                 use_wandb=args.use_wandb,
                 distill=args.distill, script_teacher=args.script_teacher, config_teacher=args.config_teacher)
    

    if name == 'main': # os.environ['CUDA_VISIBLE_DEVICES'] = "4" main() ,执行的命令是:torchrun --nproc_per_node=2 tracking/train.py --script ceutrack --config ceutrack_coesot --save_dir ./output --use_wandb 0,但是目前是仅仅在一张卡上执行两个进程,并不是使用两个GPU,然后一个进程一块GPU,现在分析问题出在哪里,然后给出修改方案。

  2. Assistant

    根据你描述的问题,原因可能是由于进程没有正确地绑定到各自的GPU上。你的命令执行了两个进程,但这两个进程可能都被绑定在了同一张GPU上,而不是分别被绑定到两张不同的GPU上。

    通常情况下,在多GPU训练时,需要确保每个进程分配到各自的GPU,这样才能正确地利用多块GPU进行训练。你提到的调度是用torchrun进行的,因此我们需要确保每个进程分配到不同的GPU。

    在第二段代码的main函数处理中,torch.distributed.launch模块会根据local_rank来设置CUDA设备。为了确保样的调度逻辑正确,需要确保每个进程的local_rank变量被正确配置并传递,从而每个进程能够正确地使用不同的GPU。

    修改后的代码

    我们可以修改你的第二段代码,确保每个进程读取并设置自己独立的local_rank。如下所示:

    import os
    import sys
    import argparse
    import importlib
    import cv2 as cv
    import torch.backends.cudnn
    import torch.distributed as dist
    
    import random
    import numpy as np
    torch.backends.cudnn.benchmark = False
    
    import _init_paths
    import lib.train.admin.settings as ws_settings
    
    def init_seeds(seed):
        random.seed(seed)
        np.random.seed(seed)
        torch.manual_seed(seed)
        torch.cuda.manual_seed(seed)
        torch.backends.cudnn.deterministic = True
        torch.backends.cudnn.benchmark = False
    
    def run_training(script_name, config_name, cudnn_benchmark=True, local_rank=-1, save_dir=None, base_seed=None,
                     use_lmdb=False, script_name_prv=None, config_name_prv=None, use_wandb=False,
                     distill=None, script_teacher=None, config_teacher=None):
        """Run the train script.
        args:
            script_name: Name of emperiment in the "experiments/" folder.
            config_name: Name of the yaml file in the "experiments/<script_name>".
            cudnn_benchmark: Use cudnn benchmark or not (default is True).
        """
        if save_dir is None:
            print("save_dir dir is not given. Use the default dir instead.")
        # This is needed to avoid strange crashes related to opencv
        cv.setNumThreads(0)
    
        torch.backends.cudnn.benchmark = cudnn_benchmark
    
        print('script_name: {}.py  config_name: {}.yaml'.format(script_name, config_name))
    
        '''2021.1.5 set seed for different process'''
        if base_seed is not None:
            if local_rank != -1:
                init_seeds(base_seed + local_rank)
            else:
                init_seeds(base_seed)
    
        settings = ws_settings.Settings()
        settings.script_name = script_name
        settings.config_name = config_name
        settings.project_path = 'train/{}/{}'.format(script_name, config_name)
        if script_name_prv is not None and config_name_prv is not None:
            settings.project_path_prv = 'train/{}/{}'.format(script_name_prv, config_name_prv)
        settings.local_rank = local_rank
        settings.save_dir = os.path.abspath(save_dir)
        settings.use_lmdb = use_lmdb
        prj_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), "../.."))
        settings.cfg_file = os.path.join(prj_dir, 'experiments/%s/%s.yaml' % (script_name, config_name))
        settings.use_wandb = use_wandb
        if distill:
            settings.distill = distill
            settings.script_teacher = script_teacher
            settings.config_teacher = config_teacher
            if script_teacher is not None and config_teacher is not None:
                settings.project_path_teacher = 'train/{}/{}'.format(script_teacher, config_teacher)
            settings.cfg_file_teacher = os.path.join(prj_dir, 'experiments/%s/%s.yaml' % (script_teacher, config_teacher))
            expr_module = importlib.import_module('lib.train.train_script_distill')
        else:
            expr_module = importlib.import_module('lib.train.train_script')
        expr_func = getattr(expr_module, 'run')
    
        expr_func(settings)
    
    def main():
        parser = argparse.ArgumentParser(description='Run a train scripts in train_settings.')
        parser.add_argument('--script', type=str, required=True, help='Name of the train script.')
        parser.add_argument('--config', type=str, required=True, help="Name of the config file.")
        parser.add_argument('--cudnn_benchmark', type=bool, default=True, help='Set cudnn benchmark on (1) or off (0) (default is on).')
        parser.add_argument('--local_rank', default=-1, type=int, help='node rank for distributed training')
        parser.add_argument('--save_dir', type=str, help='the directory to save checkpoints and logs')
        parser.add_argument('--seed', type=int, default=3407, help='seed for random numbers, default=42')
        parser.add_argument('--use_lmdb', type=int, choices=[0, 1], default=0)  # whether datasets are in lmdb format
        parser.add_argument('--script_prv', type=str, default=None, help='Name of the train script of previous model.')
        parser.add_argument('--config_prv', type=str, default=None, help="Name of the config file of previous model.")
        parser.add_argument('--use_wandb', type=int, choices=[0, 1], default=0)  # whether to use wandb
        # for knowledge distillation
        parser.add_argument('--distill', type=int, choices=[0, 1], default=0)  # whether to use knowledge distillation
        parser.add_argument('--script_teacher', type=str, help='teacher script name')
        parser.add_argument('--config_teacher', type=str, help='teacher yaml configure file name')
    
        args = parser.parse_args()
        ##############
        if 'LOCAL_RANK' in os.environ:
            args.local_rank = int(os.environ['LOCAL_RANK'])
        ####################
        if args.local_rank != -1:
            dist.init_process_group(backend='nccl')
            torch.cuda.set_device(args.local_rank)
        else:
            torch.cuda.set_device(0)
        run_training(args.script, args.config, cudnn_benchmark=args.cudnn_benchmark,
                     local_rank=args.local_rank, save_dir=args.save_dir, base_seed=args.seed,
                     use_lmdb=args.use_lmdb, script_name_prv=args.script_prv, config_name_prv=args.config_prv,
                     use_wandb=args.use_wandb,
                     distill=args.distill, script_teacher=args.script_teacher, config_teacher=args.config_teacher)
    
    if __name__ == '__main__':
        main()
    

    这里主要改动在于确保local_rank被正确读取并设置,从而每个进程绑定到不同的GPU上。并不需要额外的硬编码设置,因为torchrun会负责将local_rank作为环境变量传递,而我们在代码中读取它。

    确定使用命令

    然后,你运行分布式任务的命令应该是:

    torchrun --nproc_per_node=2 tracking/train.py --script ceutrack --config ceutrack_coesot --save_dir ./output --use_wandb 0
    

    这样应该确保两个进程分别绑定到两个GPU上。这样你就可以充分利用多GPU来加速训练了。

  3. User

    将改动的地方单独写出来

  4. Assistant

    以下是你提供的代码中需要改动的部分,以确保每个进程能够绑定到各自的 GPU 上。

    1. 添加 local_rank 环境变量读取代码

    在解析命令行参数之后,添加如下代码来读取 local_rank 环境变量:

    if 'LOCAL_RANK' in os.environ:
        args.local_rank = int(os.environ['LOCAL_RANK'])
    

    2. 根据 local_rank 设置 CUDA 设备

    在 main 函数中,修改以确保 CUDA 设备根据 local_rank 进行设置:

    if args.local_rank != -1:
        dist.init_process_group(backend='nccl')
        torch.cuda.set_device(args.local_rank)
    else:
        torch.cuda.set_device(0)
    

    改动后的集中版本

    import os
    import sys
    import argparse
    import importlib
    import cv2 as cv
    import torch.backends.cudnn
    import torch.distributed as dist
    import random
    import numpy as np
    torch.backends.cudnn.benchmark = False
    
    import _init_paths
    import lib.train.admin.settings as ws_settings
    
    def init_seeds(seed):
        random.seed(seed)
        np.random.seed(seed)
        torch.manual_seed(seed)
        torch.cuda.manual_seed(seed)
        torch.backends.cudnn.deterministic = True
        torch.backends.cudnn.benchmark = False
    
    def run_training(script_name, config_name, cudnn_benchmark=True, local_rank=-1, save_dir=None, base_seed=None,
                     use_lmdb=False, script_name_prv=None, config_name_prv=None, use_wandb=False,
                     distill=None, script_teacher=None, config_teacher=None):
        """Run the train script.
        args:
            script_name: Name of emperiment in the "experiments/" folder.
            config_name: Name of the yaml file in the "experiments/<script_name>".
            cudnn_benchmark: Use cudnn benchmark or not (default is True).
        """
        if save_dir is None:
            print("save_dir dir is not given. Use the default dir instead.")
        # This is needed to avoid strange crashes related to opencv
        cv.setNumThreads(0)
    
        torch.backends.cudnn.benchmark = cudnn_benchmark
    
        print('script_name: {}.py  config_name: {}.yaml'.format(script_name, config_name))
    
        '''2021.1.5 set seed for different process'''
        if base_seed is not None:
            if local_rank != -1:
                init_seeds(base_seed + local_rank)
            else:
                init_seeds(base_seed)
    
        settings = ws_settings.Settings()
        settings.script_name = script_name
        settings.config_name = config_name
        settings.project_path = 'train/{}/{}'.format(script_name, config_name)
        if script_name_prv is not None and config_name_prv is not None:
            settings.project_path_prv = 'train/{}/{}'.format(script_name_prv, config_name_prv)
        settings.local_rank = local_rank
        settings.save_dir = os.path.abspath(save_dir)
        settings.use_lmdb = use_lmdb
        prj_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), "../.."))
        settings.cfg_file = os.path.join(prj_dir, 'experiments/%s/%s.yaml' % (script_name, config_name))
        settings.use_wandb = use_wandb
        if distill:
            settings.distill = distill
            settings.script_teacher = script_teacher
            settings.config_teacher = config_teacher
            if script_teacher is not None and config_teacher is not None:
                settings.project_path_teacher = 'train/{}/{}'.format(script_teacher, config_teacher)
            settings.cfg_file_teacher = os.path.join(prj_dir, 'experiments/%s/%s.yaml' % (script_teacher, config_teacher))
            expr_module = importlib.import_module('lib.train.train_script_distill')
        else:
            expr_module = importlib.import_module('lib.train.train_script')
        expr_func = getattr(expr_module, 'run')
    
        expr_func(settings)
    
    def main():
        parser = argparse.ArgumentParser(description='Run a train scripts in train_settings.')
        parser.add_argument('--script', type=str, required=True, help='Name of the train script.')
        parser.add_argument('--config', type=str, required=True, help="Name of the config file.")
        parser.add_argument('--cudnn_benchmark', type=bool, default=True, help='Set cudnn benchmark on (1) or off (0) (default is on).')
        parser.add_argument('--local_rank', default=-1, type=int, help='node rank for distributed training')
        parser.add_argument('--save_dir', type=str, help='the directory to save checkpoints and logs')
        parser.add_argument('--seed', type=int, default=3407, help='seed for random numbers, default=42')
        parser.add_argument('--use_lmdb', type=int, choices=[0, 1], default=0)  # whether datasets are in lmdb format
        parser.add_argument('--script_prv', type=str, default=None, help='Name of the train script of previous model.')
        parser.add_argument('--config_prv', type=str, default=None, help="Name of the config file of previous model.")
        parser.add_argument('--use_wandb', type=int, choices=[0, 1], default=0)  # whether to use wandb
        # for knowledge distillation
        parser.add_argument('--distill', type=int, choices=[0, 1], default=0)  # whether to use knowledge distillation
        parser.add_argument('--script_teacher', type=str, help='teacher script name')
        parser.add_argument('--config_teacher', type=str, help='teacher yaml configure file name')
    
        args = parser.parse_args()
    
        # 读取 local_rank 环境变量
        if 'LOCAL_RANK' in os.environ:
            args.local_rank = int(os.environ['LOCAL_RANK'])
        
        # 设置 CUDA 设备
        if args.local_rank != -1:
            dist.init_process_group(backend='nccl')
            torch.cuda.set_device(args.local_rank)
        else:
            torch.cuda.set_device(0)
            
        run_training(args.script, args.config, cudnn_benchmark=args.cudnn_benchmark,
                     local_rank=args.local_rank, save_dir=args.save_dir, base_seed=args.seed,
                     use_lmdb=args.use_lmdb, script_name_prv=args.script_prv, config_name_prv=args.config_prv,
                     use_wandb=args.use_wandb,
                     distill=args.distill, script_teacher=args.script_teacher, config_teacher=args.config_teacher)
    
    if __name__ == '__main__':
        main()
    

    这两段改动将确保每个进程设置自己的 GPU 设备,使其能够正确地使用多张 GPU 进行训练。你可以按照新的方式执行命令以启动多 GPU 训练:

    torchrun --nproc_per_node=2 tracking/train.py --script ceutrack --config ceutrack_coesot --save_dir ./output --use_wandb 0
    

    这样,两个进程将分别使用不同的 GPU。

  5. User

    不行,还是至使用gpu0号显卡

  6. Assistant

    如果还是只使用0号显卡,问题可能在于环境变量的配置或者在代码中的设备设置逻辑不正确。为了更清楚地进行调试,我们可以采用更明确的设备分配方法。

    更改方案

    在多GPU环境下,使用torch.distributed库自动处理设备分配和通信。在实际应用中,确保每个进程绑定到不同的GPU设备上十分重要。我们可以尝试使用torch.cuda.set_device来显式地设置GPU设备。

    确保你已经安装了torch.distributed相关的包:

    pip install torch
    

    改动代码

    将以下两处加入代码确保每个进程分配到一个特定的GPU:

    Step 1: 读取local_rank环境变量

    在解析命令行参数之后添加:

    if 'LOCAL_RANK' in os.environ:
        args.local_rank = int(os.environ['LOCAL_RANK'])
    else:
        args.local_rank = 0
    

    Step 2: 配置CUDA设备和初始化进程组

    在main函数初始化设备时加入:

    if args.local_rank != -1:
        torch.cuda.set_device(args.local_rank)
        dist.init_process_group(backend='nccl')
    else:
        torch.cuda.set_device(0)
    

    修改后的代码

    完整代码如下:

    import os
    import sys
    import argparse
    import importlib
    import cv2 as cv
    import torch.backends.cudnn
    import torch.distributed as dist
    import torch
    import random
    import numpy as np
    torch.backends.cudnn.benchmark = False
    
    import _init_paths
    import lib.train.admin.settings as ws_settings
    
    def init_seeds(seed):
        random.seed(seed)
        np.random.seed(seed)
        torch.manual_seed(seed)
        torch.cuda.manual_seed(seed)
        torch.backends.cudnn.deterministic = True
        torch.backends.cudnn.benchmark = False
    
    def run_training(script_name, config_name, cudnn_benchmark=True, local_rank=-1, save_dir=None, base_seed=None,
                     use_lmdb=False, script_name_prv=None, config_name_prv=None, use_wandb=False,
                     distill=None, script_teacher=None, config_teacher=None):
        """Run the train script.
        args:
            script_name: Name of emperiment in the "experiments/" folder.
            config_name: Name of the yaml file in the "experiments/<script_name>".
            cudnn_benchmark: Use cudnn benchmark or not (default is True).
        """
        if save_dir is None:
            print("save_dir dir is not given. Use the default dir instead.")
        # This is needed to avoid strange crashes related to opencv
        cv.setNumThreads(0)
    
        torch.backends.cudnn.benchmark = cudnn_benchmark
    
        print('script_name: {}.py  config_name: {}.yaml'.format(script_name, config_name))
    
        '''2021.1.5 set seed for different process'''
        if base_seed is not None:
            if local_rank != -1:
                init_seeds(base_seed + local_rank)
            else:
                init_seeds(base_seed)
    
        settings = ws_settings.Settings()
        settings.script_name = script_name
        settings.config_name = config_name
        settings.project_path = 'train/{}/{}'.format(script_name, config_name)
        if script_name_prv is not None and config_name_prv is not None:
            settings.project_path_prv = 'train/{}/{}'.format(script_name_prv, config_name_prv)
        settings.local_rank = local_rank
        settings.save_dir = os.path.abspath(save_dir)
        settings.use_lmdb = use_lmdb
        prj_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), "../.."))
        settings.cfg_file = os.path.join(prj_dir, 'experiments/%s/%s.yaml' % (script_name, config_name))
        settings.use_wandb = use_wandb
        if distill:
            settings.distill = distill
            settings.script_teacher = script_teacher
            settings.config_teacher = config_teacher
            if script_teacher is not None and config_teacher is not None:
                settings.project_path_teacher = 'train/{}/{}'.format(script_teacher, config_teacher)
            settings.cfg_file_teacher = os.path.join(prj_dir, 'experiments/%s/%s.yaml' % (script_teacher, config_teacher))
            expr_module = importlib.import_module('lib.train.train_script_distill')
        else:
            expr_module = importlib.import_module('lib.train.train_script')
        expr_func = getattr(expr_module, 'run')
    
        expr_func(settings)
    
    def main():
        parser = argparse.ArgumentParser(description='Run a train scripts in train_settings.')
        parser.add_argument('--script', type=str, required=True, help='Name of the train script.')
        parser.add_argument('--config', type=str, required=True, help="Name of the config file.")
        parser.add_argument('--cudnn_benchmark', type=bool, default=True, help='Set cudnn benchmark on (1) or off (0) (default is on).')
        parser.add_argument('--local_rank', default=-1, type=int, help='node rank for distributed training')
        parser.add_argument('--save_dir', type=str, help='the directory to save checkpoints and logs')
        parser.add_argument('--seed', type=int, default=3407, help='seed for random numbers, default=42')
        parser.add_argument('--use_lmdb', type=int, choices=[0, 1], default=0, help="Whether datasets are in lmdb format")
        parser.add_argument('--script_prv', type=str, default=None, help='Name of the train script of previous model.')
        parser.add_argument('--config_prv', type=str, default=None, help="Name of the config file of previous model.")
        parser.add_argument('--use_wandb', type=int, choices=[0, 1], default=0, help="Whether to use wandb")
        # for knowledge distillation
        parser.add_argument('--distill', type=int, choices=[0, 1], default=0, help="Whether to use knowledge distillation")
        parser.add_argument('--script_teacher', type=str, help='Teacher script name')
        parser.add_argument('--config_teacher', type=str, help='Teacher yaml configure file name')
    
        args = parser.parse_args()
    
        # 读取local_rank环境变量
        if 'LOCAL_RANK' in os.environ:
            args.local_rank = int(os.environ['LOCAL_RANK'])
        else:
            args.local_rank = 0
    
        # 设置 CUDA 设备
        if args.local_rank != -1:
            torch.cuda.set_device(args.local_rank)
            dist.init_process_group(backend='nccl')
        else:
            torch.cuda.set_device(0)
        
        run_training(args.script, args.config, cudnn_benchmark=args.cudnn_benchmark,
                     local_rank=args.local_rank, save_dir=args.save_dir, base_seed=args.seed,
                     use_lmdb=args.use_lmdb, script_name_prv=args.script_prv, config_name_prv=args.config_prv,
                     use_wandb=args.use_wandb,
                     distill=args.distill, script_teacher=args.script_teacher, config_teacher=args.config_teacher)
    
    if __name__ == '__main__':
        main()
    

    这样,你的代码显式地读取了LOCAL_RANK环境变量并根据其值设置了对应的CUDA设备,然后初始化过程组。确保运行时使用的每个进程绑定到给定的GPU,这样你就能利用多GPU环境进行了。执行命令应如下:

    torchrun --nproc_per_node=2 tracking/train.py --script ceutrack --config ceutrack_coesot --save_dir ./output --use_wandb 0