Skip to content
TheChatArchiveThe Chat Archive
gpt-4o-2024-05-13· OpenAIAi2 WildChat-4.8M (AI2)

运行:import os import argparse import random def parseargs(): """ args for training. """…

  1. User

    运行:import os import argparse import random

    def parse_args(): """ args for training. """ parser = argparse.ArgumentParser(description='Parse args for training') # for train parser.add_argument('--script', type=str, help='training script name') parser.add_argument('--config', type=str, default='baseline', help='yaml configure file name') parser.add_argument('--save_dir', type=str, help='root directory to save checkpoints, logs, and tensorboard') parser.add_argument('--mode', type=str, choices=["single", "multiple", "multi_node"], default="multiple", help="train on single gpu or multiple gpus") # parser.add_argument('--nproc_per_node', type=int, help="number of GPUs per node") # specify when mode is multiple parser.add_argument('--nproc_per_node', type=int, default=1, help="number of GPUs per node") parser.add_argument('--use_lmdb', type=int, choices=[0, 1], default=0) # whether datasets are in lmdb format parser.add_argument('--script_prv', type=str, help='training script name') parser.add_argument('--config_prv', type=str, default='baseline', help='yaml configure file name') parser.add_argument('--use_wandb', type=int, choices=[0, 1], default=0) # whether to use wandb # for knowledge distillation parser.add_argument('--distill', type=int, choices=[0, 1], default=0) # whether to use knowledge distillation parser.add_argument('--script_teacher', type=str, help='teacher script name') parser.add_argument('--config_teacher', type=str, help='teacher yaml configure file name')

    # for multiple machines
    parser.add_argument('--rank', type=int, help='Rank of the current process.')
    parser.add_argument('--world-size', type=int, help='Number of processes participating in the job.')
    parser.add_argument('--ip', type=str, default='127.0.0.1', help='IP of the current rank 0.')
    parser.add_argument('--port', type=int, default='20000', help='Port of the current rank 0.')
    
    args = parser.parse_args()
    
    return args
    

    def main(): args = parse_args() print("Arguments parsed:", args) if args.mode == "single": train_cmd = "python lib/train/run_training.py --script %s --config %s --save_dir %s --use_lmdb %d "
    "--script_prv %s --config_prv %s --distill %d --script_teacher %s --config_teacher %s --use_wandb %d"
    % (args.script, args.config, args.save_dir, args.use_lmdb, args.script_prv, args.config_prv, args.distill, args.script_teacher, args.config_teacher, args.use_wandb) elif args.mode == "multiple": train_cmd = "torchrun --nproc_per_node 2 --master_port %d lib/train/run_training.py "
    "--script %s --config %s --save_dir %s --use_lmdb %d --script_prv %s --config_prv %s --use_wandb %d "
    "--distill %d --script_teacher %s --config_teacher %s"
    % (random.randint(10000, 50000), args.script, args.config, args.save_dir, args.use_lmdb, args.script_prv, args.config_prv, args.use_wandb, args.distill, args.script_teacher, args.config_teacher) elif args.mode == "multi_node": train_cmd = "python -m torch.distributed.launch --nproc_per_node %d --master_addr %s --master_port %d --nnodes %d --node_rank %d lib/train/run_training.py "
    "--script %s --config %s --save_dir %s --use_lmdb %d --script_prv %s --config_prv %s --use_wandb %d "
    "--distill %d --script_teacher %s --config_teacher %s"
    % (args.nproc_per_node, args.ip, args.port, args.world_size, args.rank, args.script, args.config, args.save_dir, args.use_lmdb, args.script_prv, args.config_prv, args.use_wandb, args.distill, args.script_teacher, args.config_teacher) else: raise ValueError("mode should be 'single' or 'multiple'.") print("Train command:", train_cmd) os.system(train_cmd)

    if name == "main": main() 和 import os import sys import argparse import importlib import cv2 as cv import torch.backends.cudnn import torch.distributed as dist

    import random import numpy as np torch.backends.cudnn.benchmark = False

    import _init_paths import lib.train.admin.settings as ws_settings

    def init_seeds(seed): random.seed(seed) np.random.seed(seed) torch.manual_seed(seed) torch.cuda.manual_seed(seed) torch.backends.cudnn.deterministic = True torch.backends.cudnn.benchmark = False

    def run_training(script_name, config_name, cudnn_benchmark=True, local_rank=-1, save_dir=None, base_seed=None, use_lmdb=False, script_name_prv=None, config_name_prv=None, use_wandb=False, distill=None, script_teacher=None, config_teacher=None): """Run the train script. args: script_name: Name of emperiment in the "experiments/" folder. config_name: Name of the yaml file in the "experiments/<script_name>". cudnn_benchmark: Use cudnn benchmark or not (default is True). """ if save_dir is None: print("save_dir dir is not given. Use the default dir instead.") # This is needed to avoid strange crashes related to opencv cv.setNumThreads(0)

    torch.backends.cudnn.benchmark = cudnn_benchmark
    
    print('script_name: {}.py  config_name: {}.yaml'.format(script_name, config_name))
    
    '''2021.1.5 set seed for different process'''
    if base_seed is not None:
        if local_rank != -1:
            init_seeds(base_seed + local_rank)
        else:
            init_seeds(base_seed)
    
    settings = ws_settings.Settings()
    settings.script_name = script_name
    settings.config_name = config_name
    settings.project_path = 'train/{}/{}'.format(script_name, config_name)
    if script_name_prv is not None and config_name_prv is not None:
        settings.project_path_prv = 'train/{}/{}'.format(script_name_prv, config_name_prv)
    settings.local_rank = local_rank
    settings.save_dir = os.path.abspath(save_dir)
    settings.use_lmdb = use_lmdb
    prj_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), "../.."))
    settings.cfg_file = os.path.join(prj_dir, 'experiments/%s/%s.yaml' % (script_name, config_name))
    settings.use_wandb = use_wandb
    if distill:
        settings.distill = distill
        settings.script_teacher = script_teacher
        settings.config_teacher = config_teacher
        if script_teacher is not None and config_teacher is not None:
            settings.project_path_teacher = 'train/{}/{}'.format(script_teacher, config_teacher)
        settings.cfg_file_teacher = os.path.join(prj_dir, 'experiments/%s/%s.yaml' % (script_teacher, config_teacher))
        expr_module = importlib.import_module('lib.train.train_script_distill')
    else:
        expr_module = importlib.import_module('lib.train.train_script')
    expr_func = getattr(expr_module, 'run')
    
    expr_func(settings)
    

    def main(): parser = argparse.ArgumentParser(description='Run a train scripts in train_settings.') parser.add_argument('--script', type=str, required=True, help='Name of the train script.') parser.add_argument('--config', type=str, required=True, help="Name of the config file.") parser.add_argument('--cudnn_benchmark', type=bool, default=True, help='Set cudnn benchmark on (1) or off (0) (default is on).') # parser.add_argument('--local_rank', default=-1, type=int, help='node rank for distributed training') parser.add_argument('--save_dir', type=str, help='the directory to save checkpoints and logs') parser.add_argument('--seed', type=int, default=3407, help='seed for random numbers, default=42') parser.add_argument('--use_lmdb', type=int, choices=[0, 1], default=0) # whether datasets are in lmdb format parser.add_argument('--script_prv', type=str, default=None, help='Name of the train script of previous model.') parser.add_argument('--config_prv', type=str, default=None, help="Name of the config file of previous model.") parser.add_argument('--use_wandb', type=int, choices=[0, 1], default=0) # whether to use wandb # for knowledge distillation parser.add_argument('--distill', type=int, choices=[0, 1], default=0) # whether to use knowledge distillation parser.add_argument('--script_teacher', type=str, help='teacher script name') parser.add_argument('--config_teacher', type=str, help='teacher yaml configure file name')

    args = parser.parse_args()
    print("Arguments parsed:", args)
    if 'LOCAL_RANK' in os.environ:
        args.local_rank = int(os.environ['LOCAL_RANK'])
    print(f"LOCAL_RANK from environment: {os.environ.get('LOCAL_RANK')}")
    print(f"LOCAL_RANK from args: {args.local_rank}")
        ####################
    num_gpus = torch.cuda.device_count()
    print(f"Number of GPUs available: {num_gpus}")
    
    if args.local_rank != -1:
        print(f"Initializing process group with local_rank: {args.local_rank}")
        dist.init_process_group(backend='nccl')
        torch.cuda.set_device(args.local_rank)
        print(f"Assigned GPU: {torch.cuda.current_device()}")
    else:
        print("Setting device to GPU 0")
        torch.cuda.set_device(0)
        print(f"Assigned GPU: {torch.cuda.current_device()}")
    run_training(args.script, args.config, cudnn_benchmark=args.cudnn_benchmark,
                 local_rank=args.local_rank, save_dir=args.save_dir, base_seed=args.seed,
                 use_lmdb=args.use_lmdb, script_name_prv=args.script_prv, config_name_prv=args.config_prv,
                 use_wandb=args.use_wandb,
                 distill=args.distill, script_teacher=args.script_teacher, config_teacher=args.config_teacher)
    

    if name == 'main': # os.environ['CUDA_VISIBLE_DEVICES'] = "4" main() ,打印:[2024-06-20 16:53:31,618] torch.distributed.run: [WARNING] [2024-06-20 16:53:31,618] torch.distributed.run: [WARNING] ***************************************** [2024-06-20 16:53:31,618] torch.distributed.run: [WARNING] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. [2024-06-20 16:53:31,618] torch.distributed.run: [WARNING] ***************************************** Arguments parsed:Arguments parsed: Namespace(script='ceutrack', config='ceutrack_coesot', save_dir='./output', mode='multiple', nproc_per_node=1, use_lmdb=0, script_prv=None, config_prv='baseline', use_wandb=0, distill=0, script_teacher=None, config_teacher=None, rank=None, world_size=None, ip='127.0.0.1', port=20000) Train command: torchrun --nproc_per_node 2 --master_port 18655 lib/train/run_training.py --script ceutrack --config ceutrack_coesot --save_dir ./output --use_lmdb 0 --script_prv None --config_prv baseline --use_wandb 0 --distill 0 --script_teacher None --config_teacher None Namespace(script='ceutrack', config='ceutrack_coesot', save_dir='./output', mode='multiple', nproc_per_node=1, use_lmdb=0, script_prv=None, config_prv='baseline', use_wandb=0, distill=0, script_teacher=None, config_teacher=None, rank=None, world_size=None, ip='127.0.0.1', port=20000) Train command: torchrun --nproc_per_node 2 --master_port 25391 lib/train/run_training.py --script ceutrack --config ceutrack_coesot --save_dir ./output --use_lmdb 0 --script_prv None --config_prv baseline --use_wandb 0 --distill 0 --script_teacher None --config_teacher None 2024-06-20 16:53:34.653896: I tensorflow/core/platform/cpu_feature_guard.cc:210] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations. To enable the following instructions: AVX2 AVX512F FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags. 2024-06-20 16:53:34.653898: I tensorflow/core/platform/cpu_feature_guard.cc:210] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations. To enable the following instructions: AVX2 AVX512F FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags. 2024-06-20 16:53:34.716200: I tensorflow/core/platform/cpu_feature_guard.cc:210] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations. To enable the following instructions: AVX2 AVX512F FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags. 2024-06-20 16:53:34.717588: I tensorflow/core/platform/cpu_feature_guard.cc:210] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations. To enable the following instructions: AVX2 AVX512F FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags. 2024-06-20 16:53:35.421930: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT 2024-06-20 16:53:35.422292: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT 2024-06-20 16:53:35.501211: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT 2024-06-20 16:53:35.506000: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT Arguments parsed: Namespace(script='ceutrack', config='ceutrack_coesot', cudnn_benchmark=True, save_dir='./output', seed=3407, use_lmdb=0, script_prv='None', config_prv='baseline', use_wandb=0, distill=0, script_teacher='None', config_teacher='None') LOCAL_RANK from environment: 0 LOCAL_RANK from args: 0 Arguments parsed: Namespace(script='ceutrack', config='ceutrack_coesot', cudnn_benchmark=True, save_dir='./output', seed=3407, use_lmdb=0, script_prv='None', config_prv='baseline', use_wandb=0, distill=0, script_teacher='None', config_teacher='None') LOCAL_RANK from environment: 1 LOCAL_RANK from args: 1 Number of GPUs available: 2 Initializing process group with local_rank: 0 Number of GPUs available: 2 Initializing process group with local_rank: 1 Arguments parsed: Namespace(script='ceutrack', config='ceutrack_coesot', cudnn_benchmark=True, save_dir='./output', seed=3407, use_lmdb=0, script_prv='None', config_prv='baseline', use_wandb=0, distill=0, script_teacher='None', config_teacher='None') LOCAL_RANK from environment: 0 LOCAL_RANK from args: 0 Assigned GPU: 0 script_name: ceutrack.py config_name: ceutrack_coesot.yaml Assigned GPU: 1 script_name: ceutrack.py config_name: ceutrack_coesot.yaml Number of GPUs available: 2 Initializing process group with local_rank: 0 Assigned GPU: 0 script_name: ceutrack.py config_name: ceutrack_coesot.yaml Arguments parsed: Namespace(script='ceutrack', config='ceutrack_coesot', cudnn_benchmark=True, save_dir='./output', seed=3407, use_lmdb=0, script_prv='None', config_prv='baseline', use_wandb=0, distill=0, script_teacher='None', config_teacher='None') LOCAL_RANK from environment: 1 LOCAL_RANK from args: 1 Number of GPUs available: 2 Initializing process group with local_rank: 1 Assigned GPU: 1 ,训练命令是 export CUDA_VISIBLE_DEVICES=0,1 和 torchrun --nproc_per_node=2 tracking/train.py --script ceutrack --config ceutrack_coesot --save_dir ./output --use_wandb 0 ,但是在显卡使用上,目前是两块GPU,每块GPU上分了两个进程,但是应该是一块GPU一个进程。

  2. Assistant

    根据你提供的信息,系统目前在每块GPU上都启动了两个进程,而预期行为应该是在每块GPU上启动一个进程。这种情况往往是由于多次运行了命令,或者命令参数配置不正确。

    让我们一步步审查并确保命令行参数和代码逻辑正确。

    关键点

    1. 检查 torchrun 的参数:确保你提到的设备数量和实际使用的设备数量一致。
    2. 环境变量:确保 CUDA_VISIBLE_DEVICES 配置正确。
    3. 检查 torchrun 命令是否重复执行。

    代码审查

    首先检查 torchrun 命令:

    export CUDA_VISIBLE_DEVICES=0,1
    torchrun --nproc_per_node=2 tracking/train.py --script ceutrack --config ceutrack_coesot --save_dir ./output --use_wandb 0
    

    该命令假设每块 GPU 上启动一个进程。

    确保代码逻辑正确

    你提供的第二段代码中已经进行了 local_rank 的处理:

    def main():
        ...
        if args.local_rank != -1:
            print(f"Initializing process group with local_rank: {args.local_rank}")
            dist.init_process_group(backend='nccl')
            torch.cuda.set_device(args.local_rank)
            print(f"Assigned GPU: {torch.cuda.current_device()}")
        else:
            print("Setting device to GPU 0")
            torch.cuda.set_device(0)
            print(f"Assigned GPU: {torch.cuda.current_device()}")
        ...
    

    local_rank 的设置应该由 torchrun 自动传递,能够正确设置设备。

    排查并重设环境

    1. 重置 CUDA 环境:

      停止所有 Python 进程并重启服务器以确保没有遗留的进程干扰。

    2. 确定环境变量:

      确保环境变量正确设置,执行以下命令并检查输出:

      echo $CUDA_VISIBLE_DEVICES
      

      预期输出:

      0,1
      
    3. 简化命令:

      改用 torchrun 执行命令,这有助于精准控制进程数:

      export CUDA_VISIBLE_DEVICES=0,1
      torchrun --nproc_per_node=2 tracking/train.py --script ceutrack --config ceutrack_coesot --save_dir ./output --use_wandb 0
      

      或者直接用 torchrun 手动指定进程数:

      torchrun --nnodes=1 --nproc_per_node=2 tracking/train.py --script ceutrack --config ceutrack_coesot --save_dir ./output --use_wandb 0
      
    4. 检查 Python 脚本:

      重新确认你的脚本没有意外地多次调用,例如在train.py中,将关键初始化部分放在if __name__ == "__main__":内部:

      if __name__ == '__main__':
          main()
      
    5. 捕获调试信息:

      最后,运行下面命令以捕获更多调试信息:

      TORCH_DISTRIBUTED_DEBUG=DETAIL torchrun --nnodes=1 --nproc_per_node=2 tracking/train.py --script ceutrack --config ceutrack_coesot --save_dir ./output --use_wandb 0
      

      这将提供更详细的调试信息,有助于进一步诊断问题。

    通过以上步骤逐步排查和验证,应该能解决多进程分配不正确的问题。如果问题仍然存在,请重新确认命令行调用和脚本逻辑是否有任何多余的或重复执行的部分。

  3. User

    我们把命令改为torchrun tracking/train.py --script ceutrack --config ceutrack_coesot --save_dir ./output --use_wandb 0 ,现在执行了两个gpu每个gpu一个进程

  4. Assistant

    很好,听起来你的问题已经部分解决了。每个 GPU 上运行一个进程是符合预期的。现在我们来看一下更详细的配置和优化,以确保训练过程顺利进行。

    优化建议

    1. 确保环境设置正确:在每次执行命令之前,都要确保 CUDA 环境变量设置正确。

    2. 清理之前的进程:在每次运行新命令之前,确保所有相关的 Python 进程已经终止,以防止冲突。

    3. 正确使用 Distributed Package:使用 torchrun 来管理进程,确实是目前推荐的方式,更加稳健和灵活。

    双重确认脚本配置

    确保你的 Python 脚本 train.py 中依然维持对 local_rank 正确的处理:

    def main():
        parser = argparse.ArgumentParser(description='Run a train scripts in train_settings.')
        parser.add_argument('--script', type=str, required=True, help='Name of the train script.')
        parser.add_argument('--config', type=str, required=True, help='Name of the config file.')
        parser.add_argument('--cudnn_benchmark', type=bool, default=True, help='Set cudnn benchmark on (1) or off (0) (default is on).')
        parser.add_argument('--save_dir', type=str, help='the directory to save checkpoints and logs')
        parser.add_argument('--seed', type=int, default=3407, help='seed for random numbers, default=42')
        parser.add_argument('--use_lmdb', type=int, choices=[0, 1], default=0)  # whether datasets are in lmdb format
        parser.add_argument('--script_prv', type=str, default=None, help='Name of the train script of previous model.')
        parser.add_argument('--config_prv', type=str, default=None, help="Name of the config file of previous model.")
        parser.add_argument('--use_wandb', type=int, choices=[0, 1], default=0)  # whether to use wandb
        # for knowledge distillation
        parser.add_argument('--distill', type=int, choices=[0, 1], default=0)  # whether to use knowledge distillation
        parser.add_argument('--script_teacher', type=str, help='teacher script name')
        parser.add_argument('--config_teacher', type=str, help='teacher yaml configure file name')
    
        args = parser.parse_args()
        print("Arguments parsed:", args)
        
        if 'LOCAL_RANK' in os.environ:
            args.local_rank = int(os.environ['LOCAL_RANK'])
        print(f"LOCAL_RANK from environment: {os.environ.get('LOCAL_RANK')}")
        print(f"LOCAL_RANK from args: {args.local_rank}")
    
        ####################
        num_gpus = torch.cuda.device_count()
        print(f"Number of GPUs available: {num_gpus}")
        
        if args.local_rank != -1:
            print(f"Initializing process group with local_rank: {args.local_rank}")
            dist.init_process_group(backend='nccl')
            torch.cuda.set_device(args.local_rank)
            print(f"Assigned GPU: {torch.cuda.current_device()}")
        else:
            print("Setting device to GPU 0")
            torch.cuda.set_device(0)
            print(f"Assigned GPU: {torch.cuda.current_device()}")
        
        run_training(args.script, args.config, cudnn_benchmark=args.cudnn_benchmark,
                     local_rank=args.local_rank, save_dir=args.save_dir, base_seed=args.seed,
                     use_lmdb=args.use_lmdb, script_name_prv=args.script_prv, config_name_prv=args.config_prv,
                     use_wandb=args.use_wandb,
                     distill=args.distill, script_teacher=args.script_teacher, config_teacher=args.config_teacher)
    
    if __name__ == '__main__':
        main()
    

    确保 init_seeds 和 run_training 函数定义放置在适当的位置。

    核心脚本配置

    在训练脚本中设置环境变量,使得每个进程只会使用分配给它的 GPU:

    os.environ['CUDA_VISIBLE_DEVICES'] = str(args.local_rank)
    

    在 run_training 调用之前,需要确保执行 init_process_group,即 dist.init_process_group 正确。

    最后确保执行以下命令:

    export CUDA_VISIBLE_DEVICES=0,1
    torchrun tracking/train.py --script ceutrack --config ceutrack_coesot --save_dir ./output --use_wandb 0
    

    总结

    通过审查确保你的代码合理配置,以及合理设置运行时的环境变量,应该能够确保每个 GPU 上独立运行一个进程。如果仍有问题,可考虑提供更多细节或日志,以进行进一步调试。