Skip to content
TheChatArchiveThe Chat Archive
gpt-4o-mini-2024-07-18· OpenAIAi2 WildChat-4.8M (AI2)

import os import subprocess import pytest import sys sys.path.append('./')…

  1. User

    import os import subprocess import pytest import sys sys.path.append('./') sys.path.append('../')

    from bredet.auto_verification.utils.general import merge_and_save_configs

    def ensure_log_directory(): log_dir = 'log' os.makedirs(log_dir, exist_ok=True) print(f"Ensuring log directory exists: {log_dir}")

    def generate_log_file_paths(base_name): log_dir = 'log' detailed_log_file = os.path.join(log_dir, f"{base_name}_detailed.log") return detailed_log_file

    def run_command(command, description, detailed_log_file): try: with open(detailed_log_file, 'a') as log_file: log_file.write(f"Running command: {command}\n") result = subprocess.run(command, capture_output=True, text=True, shell=True) with open(detailed_log_file, 'a') as log_file: log_file.write(f"Command output: {result.stdout}\n") log_file.write(f"Command error: {result.stderr}\n")

        if result.returncode == 0:
            print(f"Command succeeded: {command}")
        else:
            print(f"Command failed: {command}")
            with open(detailed_log_file, 'a') as log_file:
                log_file.write(f"Test: {description}\n")
                log_file.write(f"Command: {command}\n")
                log_file.write(f"Error Output: {result.stderr}\n")
                log_file.write("-" * 80 + "\n")
    
        return result.returncode == 0
    except Exception as e:
        with open(detailed_log_file, 'a') as log_file:
            log_file.write(f"{description} failed with exception: {e}\n")
            log_file.write(f"Command: {command}\n")
            log_file.write(f"Exception: {str(e)}\n")
            log_file.write("-" * 80 + "\n")
        print(f"{description} failed with exception: {e}")
        return False
    

    def get_py_files_in_subdirectories(config_path): py_files = [] for subdir in os.listdir(config_path): subdir_path = os.path.join(config_path, subdir) if os.path.isdir(subdir_path): for file in os.listdir(subdir_path): if file.endswith('.py') and ('override' not in file): py_files.append(os.path.join(subdir_path, file)) return py_files

    @pytest.fixture(scope="module") def prepare_test_files(): ensure_log_directory()

    config_path = "config/test_config"
    override_config = 'bredet/auto_verification/config/auto_config.py'
    
    py_files_list = get_py_files_in_subdirectories(config_path)
    over_list = []
    for py_file in py_files_list:
        base_name = py_file.rsplit('.', 1)[0]  # 去掉文件扩展名
        extension = py_file.rsplit('.', 1)[1]  # 获取文件扩展名
        over_py_file = f"{base_name}_override.{extension}"
        if not os.path.exists(over_py_file):
            merge_and_save_configs(py_file, override_config, over_py_file, 'test')
        over_list.append(over_py_file)
    
    return over_list
    

    @pytest.fixture def detailed_log_file(request): py_file = request.node.get_closest_marker('py_file').args[0] base_name = os.path.splitext(os.path.basename(py_file))[0] return generate_log_file_paths(base_name)

    class TestModelPipeline: def init(self, py_file): if py_file is None: raise ValueError("py_file must be provided") self.py_file = py_file base_name = os.path.splitext(os.path.basename(self.py_file))[0] self.detailed_log_file = generate_log_file_paths(base_name)

    def setup_method(self):
        # Initialize logging configuration for each test
        with open(self.detailed_log_file, 'a') as log_file:
            log_file.write(f"Starting test with config: {self.py_file}\n")
    
    def teardown_method(self):
        with open(self.detailed_log_file, 'a') as log_file:
            log_file.write(f"Ending test with config: {self.py_file}\n")
    
    def test_single_test(self):   # 单卡测试
        command = f"python tools/test.py --config {self.py_file}"
        description = "Single card test"
        result = run_command(command, description, self.detailed_log_file)
        assert result, f"{description} failed"
    
    def test_multi_devices_test(self):   # 多卡测试
        command = f"torchrun tools/test.py --config {self.py_file}"
        description = "Multi-device test"
        result = run_command(command, description, self.detailed_log_file)
        assert result, f"{description} failed"
    

    def test_pipeline(py_file, detailed_log_file): test_model_pipeline = TestModelPipeline(py_file) test_model_pipeline.setup_method() test_model_pipeline.test_single_test() test_model_pipeline.test_multi_devices_test() test_model_pipeline.teardown_method()

    if name == 'main': pytest.main()报错了 帮我修改一下

  2. Assistant

    在你给出的代码中,有几个潜在的问题和改进之处。我会逐一列出并对代码进行必要的修改。

    1. pytest 的 fixture 使用不当

    你的 test_pipeline 函数应该标记为 pytest 的测试用例,并且需要处理 fixture 的使用。现在,它似乎在试图直接调用一个构造函数而没有合适的上下文。

    2. TestModelPipeline 类没有正确初始化

    在 test_pipeline 中,一个 TestModelPipeline 实例被创建,但是没有使用 pytest 的 fixture 正确传递 py_file 。这可能导致测试失败。

    3. Pytest 参数化

    可以使用 pytest 的参数化功能来逐个文件地运行测试,避免了在同一个测试用例中手动管理多实例。

    修改后的代码

    import os
    import subprocess
    import pytest
    import sys
    sys.path.append('./')
    sys.path.append('../')
    
    from bredet.auto_verification.utils.general import merge_and_save_configs
    
    def ensure_log_directory():
        log_dir = 'log'
        os.makedirs(log_dir, exist_ok=True)
        print(f"Ensuring log directory exists: {log_dir}")
    
    def generate_log_file_paths(base_name):
        log_dir = 'log'
        detailed_log_file = os.path.join(log_dir, f"{base_name}_detailed.log")
        return detailed_log_file
    
    def run_command(command, description, detailed_log_file):
        try:
            with open(detailed_log_file, 'a') as log_file:
                log_file.write(f"Running command: {command}\n")
            result = subprocess.run(command, capture_output=True, text=True, shell=True)
            with open(detailed_log_file, 'a') as log_file:
                log_file.write(f"Command output: {result.stdout}\n")
                log_file.write(f"Command error: {result.stderr}\n")
    
            if result.returncode == 0:
                print(f"Command succeeded: {command}")
                return True
            else:
                print(f"Command failed: {command}")
                with open(detailed_log_file, 'a') as log_file:
                    log_file.write(f"Test: {description}\n")
                    log_file.write(f"Command: {command}\n")
                    log_file.write(f"Error Output: {result.stderr}\n")
                    log_file.write("-" * 80 + "\n")
                return False
        except Exception as e:
            with open(detailed_log_file, 'a') as log_file:
                log_file.write(f"{description} failed with exception: {e}\n")
                log_file.write(f"Command: {command}\n")
                log_file.write(f"Exception: {str(e)}\n")
                log_file.write("-" * 80 + "\n")
            print(f"{description} failed with exception: {e}")
            return False
    
    def get_py_files_in_subdirectories(config_path):
        py_files = []
        for subdir in os.listdir(config_path):
            subdir_path = os.path.join(config_path, subdir)
            if os.path.isdir(subdir_path):
                for file in os.listdir(subdir_path):
                    if file.endswith('.py') and ('override' not in file):
                        py_files.append(os.path.join(subdir_path, file))
        return py_files
    
    @pytest.fixture(scope="module")
    def prepare_test_files():
        ensure_log_directory()
    
        config_path = "config/test_config"
        override_config = 'bredet/auto_verification/config/auto_config.py'
    
        py_files_list = get_py_files_in_subdirectories(config_path)
        over_list = []
        for py_file in py_files_list:
            base_name = py_file.rsplit('.', 1)[0]
            extension = py_file.rsplit('.', 1)[-1]
            over_py_file = f"{base_name}_override.{extension}"
            if not os.path.exists(over_py_file):
                merge_and_save_configs(py_file, override_config, over_py_file, 'test')
            over_list.append(over_py_file)
    
        return over_list
    
    @pytest.fixture
    def detailed_log_file(request):
        py_file = request.param
        base_name = os.path.splitext(os.path.basename(py_file))[0]
        return generate_log_file_paths(base_name)
    
    class TestModelPipeline:
        def __init__(self, py_file):
            if py_file is None:
                raise ValueError("py_file must be provided")
            self.py_file = py_file
            base_name = os.path.splitext(os.path.basename(self.py_file))[0]
            self.detailed_log_file = generate_log_file_paths(base_name)
    
        def setup_method(self):
            with open(self.detailed_log_file, 'a') as log_file:
                log_file.write(f"Starting test with config: {self.py_file}\n")
    
        def teardown_method(self):
            with open(self.detailed_log_file, 'a') as log_file:
                log_file.write(f"Ending test with config: {self.py_file}\n")
    
        def test_single_test(self):
            command = f"python tools/test.py --config {self.py_file}"
            description = "Single card test"
            result = run_command(command, description, self.detailed_log_file)
            assert result, f"{description} failed"
    
        def test_multi_devices_test(self):
            command = f"torchrun tools/test.py --config {self.py_file}"
            description = "Multi-device test"
            result = run_command(command, description, self.detailed_log_file)
            assert result, f"{description} failed"
    
    @pytest.mark.parametrize("detailed_log_file", ['detailed_log_file'], indirect=True)
    @pytest.mark.parametrize("py_file", prepare_test_files(), indirect=True)
    def test_pipeline(py_file, detailed_log_file):
        test_model_pipeline = TestModelPipeline(py_file)
        test_model_pipeline.setup_method()
        test_model_pipeline.test_single_test()
        test_model_pipeline.test_multi_devices_test()
        test_model_pipeline.teardown_method()
    
    if __name__ == '__main__':
        pytest.main()
    

    关键的修改说明:

    1. 使用 @pytest.mark.parametrize 来处理测试文件和日志文件的传递。
    2. 修正了 detailed_log_file fixture 使其能够正确处理测试文件。
    3. 调整了 test_pipeline 的定义,使其能够运行 pytest 测试。

    这样修改之后,你的测试用例应该可以正常运行。如果还有其他报错,请提供具体的错误信息,以便进一步帮助你。

  3. User

    Traceback (most recent call last): File "bredet/auto_verification/auto_verfication_test.py", line 113, in @pytest.mark.parametrize("py_file", prepare_test_files(), indirect=True) File "/root/miniconda3/envs/bredet2/lib/python3.8/site-packages/_pytest/fixtures.py", line 1169, in result fail(message, pytrace=False) File "/root/miniconda3/envs/bredet2/lib/python3.8/site-packages/_pytest/outcomes.py", line 178, in fail raise Failed(msg=reason, pytrace=pytrace) Failed: Fixture "prepare_test_files" called directly. Fixtures are not meant to be called directly, but are created automatically when test functions request them as parameters. See https://docs.pytest.org/en/stable/explanation/fixtures.html for more information about fixtures, and https://docs.pytest.org/en/stable/deprecations.html#calling-fixtures-directly about how to update your co

  4. Assistant

    根据你提供的错误信息,prepare_test_files 撒应该被视为一个 pytest fixture,你不应该直接调用它。在使用 pytest fixtures 时,它们应该通过参数传递,而不是被直接调用。

    修改建议

    1. 使用 pytest fixture 来获取测试文件:将 prepare_test_files 作为一个 fixture,直接作为测试函数的参数来使用,而不是尝试在 @pytest.mark.parametrize 中使用它。

    2. 调整 test_pipeline 函数:将其改为接受作为参数传递的 py_file 和 detailed_log_file 的 fixture。

    下面是修改后的示例代码:

    import os
    import subprocess
    import pytest
    import sys
    
    sys.path.append('./')
    sys.path.append('../')
    
    from bredet.auto_verification.utils.general import merge_and_save_configs
    
    def ensure_log_directory():
        log_dir = 'log'
        os.makedirs(log_dir, exist_ok=True)
        print(f"Ensuring log directory exists: {log_dir}")
    
    def generate_log_file_paths(base_name):
        log_dir = 'log'
        detailed_log_file = os.path.join(log_dir, f"{base_name}_detailed.log")
        return detailed_log_file
    
    def run_command(command, description, detailed_log_file):
        try:
            with open(detailed_log_file, 'a') as log_file:
                log_file.write(f"Running command: {command}\n")
            result = subprocess.run(command, capture_output=True, text=True, shell=True)
            with open(detailed_log_file, 'a') as log_file:
                log_file.write(f"Command output: {result.stdout}\n")
                log_file.write(f"Command error: {result.stderr}\n")
    
            if result.returncode == 0:
                print(f"Command succeeded: {command}")
                return True
            else:
                print(f"Command failed: {command}")
                with open(detailed_log_file, 'a') as log_file:
                    log_file.write(f"Test: {description}\n")
                    log_file.write(f"Command: {command}\n")
                    log_file.write(f"Error Output: {result.stderr}\n")
                    log_file.write("-" * 80 + "\n")
                return False
        except Exception as e:
            with open(detailed_log_file, 'a') as log_file:
                log_file.write(f"{description} failed with exception: {e}\n")
                log_file.write(f"Command: {command}\n")
                log_file.write(f"Exception: {str(e)}\n")
                log_file.write("-" * 80 + "\n")
            print(f"{description} failed with exception: {e}")
            return False
    
    def get_py_files_in_subdirectories(config_path):
        py_files = []
        for subdir in os.listdir(config_path):
            subdir_path = os.path.join(config_path, subdir)
            if os.path.isdir(subdir_path):
                for file in os.listdir(subdir_path):
                    if file.endswith('.py') and ('override' not in file):
                        py_files.append(os.path.join(subdir_path, file))
        return py_files
    
    @pytest.fixture(scope="module")
    def prepare_test_files():
        ensure_log_directory()
    
        config_path = "config/test_config"
        override_config = 'bredet/auto_verification/config/auto_config.py'
    
        py_files_list = get_py_files_in_subdirectories(config_path)
        over_list = []
        for py_file in py_files_list:
            base_name = py_file.rsplit('.', 1)[0]
            extension = py_file.rsplit('.', 1)[-1]
            over_py_file = f"{base_name}_override.{extension}"
            if not os.path.exists(over_py_file):
                merge_and_save_configs(py_file, override_config, over_py_file, 'test')
            over_list.append(over_py_file)
    
        return over_list
    
    @pytest.fixture
    def detailed_log_file(request):
        py_file = request.param
        base_name = os.path.splitext(os.path.basename(py_file))[0]
        return generate_log_file_paths(base_name)
    
    class TestModelPipeline:
        def __init__(self, py_file):
            if py_file is None:
                raise ValueError("py_file must be provided")
            self.py_file = py_file
            base_name = os.path.splitext(os.path.basename(self.py_file))[0]
            self.detailed_log_file = generate_log_file_paths(base_name)
    
        def setup_method(self):
            with open(self.detailed_log_file, 'a') as log_file:
                log_file.write(f"Starting test with config: {self.py_file}\n")
    
        def teardown_method(self):
            with open(self.detailed_log_file, 'a') as log_file:
                log_file.write(f"Ending test with config: {self.py_file}\n")
    
        def test_single_test(self):
            command = f"python tools/test.py --config {self.py_file}"
            description = "Single card test"
            result = run_command(command, description, self.detailed_log_file)
            assert result, f"{description} failed"
    
        def test_multi_devices_test(self):
            command = f"torchrun tools/test.py --config {self.py_file}"
            description = "Multi-device test"
            result = run_command(command, description, self.detailed_log_file)
            assert result, f"{description} failed"
    
    def test_pipeline(prepare_test_files, detailed_log_file):
        for py_file in prepare_test_files:
            # We pass the current `py_file` to the TestModelPipeline
            test_model_pipeline = TestModelPipeline(py_file)
            test_model_pipeline.setup_method()
            test_model_pipeline.test_single_test()
            test_model_pipeline.test_multi_devices_test()
            test_model_pipeline.teardown_method()
    
    if __name__ == '__main__':
        pytest.main()
    

    更改说明:

    1. 固定了 test_pipeline:test_pipeline 函数现在直接接受 prepare_test_files fixture 作为参数。这个 fixture 获取所有待测试的 Python 文件。
    2. ** 移除 @pytest.mark.parametrize**:不再使用 @pytest.mark.parametrize 而是通过循环处理 prepare_test_files 返回的文件列表。

    这样修改后,代码不应该再出现你之前遇到的错误。如果你的测试仍然失败,确保你的路径和文件存在,并提供详细的错误信息以便进一步排查问题。

  5. User

    又报错了 =========================================================== ERRORS ============================================================ ______________________________________ ERROR collecting 11_unit/auto_verfication_test.py ______________________________________ 11_unit/auto_verfication_test.py:61: in init raise ValueError("py_file must be provided") E ValueError: py_file must be provided _____________________________ ERROR collecting bredet/auto_verification/auto_verfication_test.py ______________________________ import file mismatch: imported module 'auto_verfication_test' has this file attribute: /mediap1/liuken/bretrain_auto/breseedetection20241127/breseedetection/11_unit/auto_verfication_test.py which is not the same as the test file we want to collect: /mediap1/liuken/bretrain_auto/breseedetection20241127/breseedetection/bredet/auto_verification/auto_verfication_test.py HINT: remove pycache / .pyc files and/or use a unique basename for your test file modules =================================================== short test summary info =================================================== ERROR 11_unit/auto_verfication_test.py::TestModelPipeline - ValueError: py_file must be provided ERROR bredet/auto_verification/auto_verfication_test.py !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! Interru

  6. Assistant

    从你提供的错误信息来看,主要有两个问题:

    1. ValueError: py_file must be provided:这表示在TestModelPipeline类的初始化过程中,没有提供py_file参数。这很可能是由于在test_pipeline函数中处理文件的方式不正确导致的。

    2. 导入错误:错误信息中提到的导入文件不匹配问题,通常会发生在同一目录下存在同名文件时或文件的路径被修改而导致的。

    针对这两个问题的解决方案

    1. 确保正确传递参数:在test_pipeline函数中,要确保在循环中每次都正确地创建TestModelPipeline的实例,并为其提供py_file参数。

    2. 修复导入错误:确保所有相关的 .py 文件路径无冲突,且没有在不同的地方导入同样的模块。此外,清理 _pycache_ 目录及 .pyc 文件,以确保没有旧的缓存数据影响测试运行。

    修改后的代码示例

    下面是修改后的代码,更加关注了参数的传递,确保能够正确运行:

    import os
    import subprocess
    import pytest
    import sys
    
    sys.path.append('./')
    sys.path.append('../')
    
    from bredet.auto_verification.utils.general import merge_and_save_configs
    
    def ensure_log_directory():
        log_dir = 'log'
        os.makedirs(log_dir, exist_ok=True)
        print(f"Ensuring log directory exists: {log_dir}")
    
    def generate_log_file_paths(base_name):
        log_dir = 'log'
        detailed_log_file = os.path.join(log_dir, f"{base_name}_detailed.log")
        return detailed_log_file
    
    def run_command(command, description, detailed_log_file):
        try:
            with open(detailed_log_file, 'a') as log_file:
                log_file.write(f"Running command: {command}\n")
            result = subprocess.run(command, capture_output=True, text=True, shell=True)
            with open(detailed_log_file, 'a') as log_file:
                log_file.write(f"Command output: {result.stdout}\n")
                log_file.write(f"Command error: {result.stderr}\n")
    
            if result.returncode == 0:
                print(f"Command succeeded: {command}")
                return True
            else:
                print(f"Command failed: {command}")
                with open(detailed_log_file, 'a') as log_file:
                    log_file.write(f"Test: {description}\n")
                    log_file.write(f"Command: {command}\n")
                    log_file.write(f"Error Output: {result.stderr}\n")
                    log_file.write("-" * 80 + "\n")
                return False
        except Exception as e:
            with open(detailed_log_file, 'a') as log_file:
                log_file.write(f"{description} failed with exception: {e}\n")
                log_file.write(f"Command: {command}\n")
                log_file.write(f"Exception: {str(e)}\n")
                log_file.write("-" * 80 + "\n")
            print(f"{description} failed with exception: {e}")
            return False
    
    def get_py_files_in_subdirectories(config_path):
        py_files = []
        for subdir in os.listdir(config_path):
            subdir_path = os.path.join(config_path, subdir)
            if os.path.isdir(subdir_path):
                for file in os.listdir(subdir_path):
                    if file.endswith('.py') and ('override' not in file):
                        py_files.append(os.path.join(subdir_path, file))
        return py_files
    
    @pytest.fixture(scope="module")
    def prepare_test_files():
        ensure_log_directory()
    
        config_path = "config/test_config"
        override_config = 'bredet/auto_verification/config/auto_config.py'
    
        py_files_list = get_py_files_in_subdirectories(config_path)
        over_list = []
        for py_file in py_files_list:
            base_name = py_file.rsplit('.', 1)[0]
            extension = py_file.rsplit('.', 1)[-1]
            over_py_file = f"{base_name}_override.{extension}"
            if not os.path.exists(over_py_file):
                merge_and_save_configs(py_file, override_config, over_py_file, 'test')
            over_list.append(over_py_file)
    
        return over_list
    
    @pytest.fixture
    def detailed_log_file(request):
        py_file = request.param
        base_name = os.path.splitext(os.path.basename(py_file))[0]
        return generate_log_file_paths(base_name)
    
    class TestModelPipeline:
        def __init__(self, py_file):
            if py_file is None:
                raise ValueError("py_file must be provided")
            self.py_file = py_file
            base_name = os.path.splitext(os.path.basename(self.py_file))[0]
            self.detailed_log_file = generate_log_file_paths(base_name)
    
        def setup_method(self):
            with open(self.detailed_log_file, 'a') as log_file:
                log_file.write(f"Starting test with config: {self.py_file}\n")
    
        def teardown_method(self):
            with open(self.detailed_log_file, 'a') as log_file:
                log_file.write(f"Ending test with config: {self.py_file}\n")
    
        def test_single_test(self):
            command = f"python tools/test.py --config {self.py_file}"
            description = "Single card test"
            result = run_command(command, description, self.detailed_log_file)
            assert result, f"{description} failed"
    
        def test_multi_devices_test(self):
            command = f"torchrun tools/test.py --config {self.py_file}"
            description = "Multi-device test"
            result = run_command(command, description, self.detailed_log_file)
            assert result, f"{description} failed"
    
    def test_pipeline(prepare_test_files):
        for py_file in prepare_test_files:
            detailed_log_file_path = generate_log_file_paths(os.path.splitext(os.path.basename(py_file))[0])
            test_model_pipeline = TestModelPipeline(py_file)
            test_model_pipeline.setup_method()
            test_model_pipeline.test_single_test()
            test_model_pipeline.test_multi_devices_test()
            test_model_pipeline.teardown_method()
    
    if __name__ == '__main__':
        pytest.main()
    

    关键的修改点:

    1. 确保文件传递:在 test_pipeline 函数中,现在通过迭代 prepare_test_files 来确保 py_file 被正确传递给 TestModelPipeline 进行初始化。

    2. 解决导入错误:删除所有相关的 __pycache__ 目录和 .pyc 文件,以消除因文件导入冲突引发的错误。

    3. 在 test_pipeline 中创建 detailed_log_file_path:确保在每个测试开始时创建相应的日志文件路径。

    请尝试这些修改,并确保没有其他地方存在名称冲突或路径问题。如果问题依然存在,请提供最新的错误信息以便进一步分析。

  7. User

    detailed_log_file_path = generate_log_file_paths(os.path.splitext(os.path.basename(py_file))[0])这个没有用到 还有我希望遇到错误的测试可以继续测试,别停止

  8. Assistant

    理解你的需求后,我对代码进行了调整,以确保以下两点:

    1. 记录日志文件:删除了不必要的 detailed_log_file_path 定义。日志文件的生成和传递直接发生在TestModelPipeline实例中。

    2. 测试失败后继续执行:更新了pytest中的测试用例,以便即使某些测试失败,也能继续执行后续的测试。这可以通过捕获异常和继续执行完成。

    修改后的代码

    以下是更新后的代码:

    import os
    import subprocess
    import pytest
    import sys
    
    sys.path.append('./')
    sys.path.append('../')
    
    from bredet.auto_verification.utils.general import merge_and_save_configs
    
    def ensure_log_directory():
        log_dir = 'log'
        os.makedirs(log_dir, exist_ok=True)
        print(f"Ensuring log directory exists: {log_dir}")
    
    def generate_log_file_paths(base_name):
        log_dir = 'log'
        detailed_log_file = os.path.join(log_dir, f"{base_name}_detailed.log")
        return detailed_log_file
    
    def run_command(command, description, detailed_log_file):
        try:
            with open(detailed_log_file, 'a') as log_file:
                log_file.write(f"Running command: {command}\n")
            result = subprocess.run(command, capture_output=True, text=True, shell=True)
            with open(detailed_log_file, 'a') as log_file:
                log_file.write(f"Command output: {result.stdout}\n")
                log_file.write(f"Command error: {result.stderr}\n")
    
            if result.returncode == 0:
                print(f"Command succeeded: {command}")
                return True
            else:
                print(f"Command failed: {command}")
                with open(detailed_log_file, 'a') as log_file:
                    log_file.write(f"Test: {description}\n")
                    log_file.write(f"Command: {command}\n")
                    log_file.write(f"Error Output: {result.stderr}\n")
                    log_file.write("-" * 80 + "\n")
                return False
        except Exception as e:
            with open(detailed_log_file, 'a') as log_file:
                log_file.write(f"{description} failed with exception: {e}\n")
                log_file.write(f"Command: {command}\n")
                log_file.write(f"Exception: {str(e)}\n")
                log_file.write("-" * 80 + "\n")
            print(f"{description} failed with exception: {e}")
            return False
    
    def get_py_files_in_subdirectories(config_path):
        py_files = []
        for subdir in os.listdir(config_path):
            subdir_path = os.path.join(config_path, subdir)
            if os.path.isdir(subdir_path):
                for file in os.listdir(subdir_path):
                    if file.endswith('.py') and ('override' not in file):
                        py_files.append(os.path.join(subdir_path, file))
        return py_files
    
    @pytest.fixture(scope="module")
    def prepare_test_files():
        ensure_log_directory()
    
        config_path = "config/test_config"
        override_config = 'bredet/auto_verification/config/auto_config.py'
    
        py_files_list = get_py_files_in_subdirectories(config_path)
        over_list = []
        for py_file in py_files_list:
            base_name = py_file.rsplit('.', 1)[0]
            extension = py_file.rsplit('.', 1)[-1]
            over_py_file = f"{base_name}_override.{extension}"
            if not os.path.exists(over_py_file):
                merge_and_save_configs(py_file, override_config, over_py_file, 'test')
            over_list.append(over_py_file)
    
        return over_list
    
    @pytest.fixture
    def detailed_log_file(request):
        py_file = request.param
        base_name = os.path.splitext(os.path.basename(py_file))[0]
        return generate_log_file_paths(base_name)
    
    class TestModelPipeline:
        def __init__(self, py_file):
            if py_file is None:
                raise ValueError("py_file must be provided")
            self.py_file = py_file
            base_name = os.path.splitext(os.path.basename(self.py_file))[0]
            self.detailed_log_file = generate_log_file_paths(base_name)
    
        def setup_method(self):
            with open(self.detailed_log_file, 'a') as log_file:
                log_file.write(f"Starting test with config: {self.py_file}\n")
    
        def teardown_method(self):
            with open(self.detailed_log_file, 'a') as log_file:
                log_file.write(f"Ending test with config: {self.py_file}\n")
    
        def test_single_test(self):
            command = f"python tools/test.py --config {self.py_file}"
            description = "Single card test"
            result = run_command(command, description, self.detailed_log_file)
            assert result, f"{description} failed"
    
        def test_multi_devices_test(self):
            command = f"torchrun tools/test.py --config {self.py_file}"
            description = "Multi-device test"
            result = run_command(command, description, self.detailed_log_file)
            assert result, f"{description} failed"
    
    def test_pipeline(prepare_test_files):
        results = []
        for py_file in prepare_test_files:
            test_model_pipeline = TestModelPipeline(py_file)
            test_model_pipeline.setup_method()
            
            try:
                test_model_pipeline.test_single_test()
            except AssertionError as e:
                print(f"Test single test failed for {py_file}: {e}")
                results.append(False)
            
            try:
                test_model_pipeline.test_multi_devices_test()
            except AssertionError as e:
                print(f"Test multi-devices test failed for {py_file}: {e}")
                results.append(False)
            
            test_model_pipeline.teardown_method()
        
        # If we want, we can fail the test based on results, for example:
        assert not any(results), "Some tests failed"
    
    if __name__ == '__main__':
        pytest.main()
    

    有关更改的说明:

    1. log_file_path 移除: 移除了不使用的 detailed_log_file_path 定义。

    2. 捕获异常并继续: 在 test_pipeline 中,每个测试调用都用 try-except 包裹,这样即使一个单独的测试失败,后续测试仍会继续执行。

    3. 结果记录: 将结果添加到 results 列表中,并在最后通过 assert 检查是否有任何测试失败。如果你希望在某些测试失败时也标记整个测试为失败,最后的 assert 可以启用。

    继续执行

    这样无论是test_single_test 还是 test_multi_devices_test 其中一个失败,都不会影响其他测试的执行。如果你希望在每次测试失败时输出更详细的信息,可以根据需求再添加更多的打印或日志记录。

  9. User

    打印每个测试的结果,比如执行命令是什么,执行成功还是失败了

  10. Assistant

    要打印每个测试的结果,包括执行的命令、结果是否成功等信息,可以在每个测试方法中添加相关的打印语句。我们可以在 test_single_test 和 test_multi_devices_test 方法中输出详细信息,包括命令和执行结果。

    以下是更新后的代码,包含打印每个测试的结果:

    import os
    import subprocess
    import pytest
    import sys
    
    sys.path.append('./')
    sys.path.append('../')
    
    from bredet.auto_verification.utils.general import merge_and_save_configs
    
    def ensure_log_directory():
        log_dir = 'log'
        os.makedirs(log_dir, exist_ok=True)
        print(f"Ensuring log directory exists: {log_dir}")
    
    def generate_log_file_paths(base_name):
        log_dir = 'log'
        detailed_log_file = os.path.join(log_dir, f"{base_name}_detailed.log")
        return detailed_log_file
    
    def run_command(command, description, detailed_log_file):
        try:
            with open(detailed_log_file, 'a') as log_file:
                log_file.write(f"Running command: {command}\n")
            result = subprocess.run(command, capture_output=True, text=True, shell=True)
            with open(detailed_log_file, 'a') as log_file:
                log_file.write(f"Command output: {result.stdout}\n")
                log_file.write(f"Command error: {result.stderr}\n")
    
            if result.returncode == 0:
                print(f"Command succeeded: {command}")
                return True
            else:
                print(f"Command failed: {command}")
                with open(detailed_log_file, 'a') as log_file:
                    log_file.write(f"Test: {description}\n")
                    log_file.write(f"Command: {command}\n")
                    log_file.write(f"Error Output: {result.stderr}\n")
                    log_file.write("-" * 80 + "\n")
                return False
        except Exception as e:
            with open(detailed_log_file, 'a') as log_file:
                log_file.write(f"{description} failed with exception: {e}\n")
                log_file.write(f"Command: {command}\n")
                log_file.write(f"Exception: {str(e)}\n")
                log_file.write("-" * 80 + "\n")
            print(f"{description} failed with exception: {e}")
            return False
    
    def get_py_files_in_subdirectories(config_path):
        py_files = []
        for subdir in os.listdir(config_path):
            subdir_path = os.path.join(config_path, subdir)
            if os.path.isdir(subdir_path):
                for file in os.listdir(subdir_path):
                    if file.endswith('.py') and ('override' not in file):
                        py_files.append(os.path.join(subdir_path, file))
        return py_files
    
    @pytest.fixture(scope="module")
    def prepare_test_files():
        ensure_log_directory()
    
        config_path = "config/test_config"
        override_config = 'bredet/auto_verification/config/auto_config.py'
    
        py_files_list = get_py_files_in_subdirectories(config_path)
        over_list = []
        for py_file in py_files_list:
            base_name = py_file.rsplit('.', 1)[0]
            extension = py_file.rsplit('.', 1)[-1]
            over_py_file = f"{base_name}_override.{extension}"
            if not os.path.exists(over_py_file):
                merge_and_save_configs(py_file, override_config, over_py_file, 'test')
            over_list.append(over_py_file)
    
        return over_list
    
    @pytest.fixture
    def detailed_log_file(request):
        py_file = request.param
        base_name = os.path.splitext(os.path.basename(py_file))[0]
        return generate_log_file_paths(base_name)
    
    class TestModelPipeline:
        def __init__(self, py_file):
            if py_file is None:
                raise ValueError("py_file must be provided")
            self.py_file = py_file
            base_name = os.path.splitext(os.path.basename(self.py_file))[0]
            self.detailed_log_file = generate_log_file_paths(base_name)
    
        def setup_method(self):
            with open(self.detailed_log_file, 'a') as log_file:
                log_file.write(f"Starting test with config: {self.py_file}\n")
            print(f"Starting test with config: {self.py_file}")
    
        def teardown_method(self):
            with open(self.detailed_log_file, 'a') as log_file:
                log_file.write(f"Ending test with config: {self.py_file}\n")
            print(f"Ending test with config: {self.py_file}")
    
        def test_single_test(self):
            command = f"python tools/test.py --config {self.py_file}"
            description = "Single card test"
            result = run_command(command, description, self.detailed_log_file)
            
            # Print the test result
            if result:
                print(f"RESULT: {description} for {self.py_file} succeeded.")
            else:
                print(f"RESULT: {description} for {self.py_file} failed.")
    
        def test_multi_devices_test(self):
            command = f"torchrun tools/test.py --config {self.py_file}"
            description = "Multi-device test"
            result = run_command(command, description, self.detailed_log_file)
    
            # Print the test result
            if result:
                print(f"RESULT: {description} for {self.py_file} succeeded.")
            else:
                print(f"RESULT: {description} for {self.py_file} failed.")
    
    def test_pipeline(prepare_test_files):
        for py_file in prepare_test_files:
            test_model_pipeline = TestModelPipeline(py_file)
            test_model_pipeline.setup_method()
            
            try:
                test_model_pipeline.test_single_test()
            except Exception as e:
                print(f"Error during single test for {py_file}: {e}")
            
            try:
                test_model_pipeline.test_multi_devices_test()
            except Exception as e:
                print(f"Error during multi-device test for {py_file}: {e}")
            
            test_model_pipeline.teardown_method()
    
    if __name__ == '__main__':
        pytest.main()
    

    主要修改点:

    1. 打印每个测试的开始与结束: 在 setup_method 和 teardown_method 中加入了打印语句,记录开始和结束的状态以及相关的配置文件。

    2. 打印每个测试的结果: 在 test_single_test 和 test_multi_devices_test 方法中,增加了打印语句,显示执行的命令及其成功或失败状态。

    3. 捕获异常并打印错误信息: 在 test_pipeline 中捕获任何异常,并打印提示信息,确保即使某些测试失败,仍然会继续执行后面的测试。

    这样的实现能让你在执行测试时快速了解每个测试的结果,以及具体运行了哪些命令。如果遇到错误,该信息将在控制台中及时反馈。