decode this error for me please , 570 ) 572 return engine File…
- User
decode this error for me please , 570 ) 572 return engine
File ~/lalb/vllm/vllm/engine/llm_engine.py:325, in LLMEngine.init(self, model_config, cache_config, parallel_config, scheduler_config, device_config, load_config, lora_config, speculative_config, decoding_config, observability_config, prompt_adapter_config, executor_class, log_stats, usage_context, stat_loggers, input_registry, use_cached_outputs) 321 self.input_registry = input_registry 322 self.input_processor = input_registry.create_input_processor( 323 model_config) --> 325 self.model_executor = executor_class( 326 model_config=model_config, 327 cache_config=cache_config, 328 parallel_config=parallel_config, 329 scheduler_config=scheduler_config, 330 device_config=device_config, 331 lora_config=lora_config, 332 speculative_config=speculative_config, 333 load_config=load_config, 334 prompt_adapter_config=prompt_adapter_config, 335 observability_config=self.observability_config, 336 ) 338 if not self.model_config.embedding_mode: 339 self._initialize_kv_caches()
File ~/lalb/vllm/vllm/executor/ray_tpu_executor.py:39, in RayTPUExecutor.init(self, *args, **kwargs) 35 # Updated by implementations that require additional args to be passed 36 # to the _run_workers execute_model call 37 self.extra_execute_model_run_workers_kwargs: Dict[str, Any] = {} ---> 39 super().init(*args, **kwargs)
File ~/lalb/vllm/vllm/executor/executor_base.py:47, in ExecutorBase.init(self, model_config, cache_config, parallel_config, scheduler_config, device_config, load_config, lora_config, speculative_config, prompt_adapter_config, observability_config) 45 self.prompt_adapter_config = prompt_adapter_config 46 self.observability_config = observability_config ---> 47 self._init_executor()
File ~/lalb/vllm/vllm/executor/ray_tpu_executor.py:51, in RayTPUExecutor._init_executor(self) 48 os.environ["RAY_USAGE_STATS_ENABLED"] = "0" 50 # Create the parallel TPU workers. ---> 51 self._init_workers_ray(placement_group)
File ~/lalb/vllm/vllm/executor/ray_tpu_executor.py:197, in RayTPUExecutor._init_workers_ray(self, placement_group, **ray_remote_kwargs) 188 init_worker_all_kwargs = [ 189 self._get_worker_kwargs( 190 local_rank=node_workers[node_id].index(rank), (...) 193 ) for rank, (node_id, _) in enumerate(worker_node_and_gpu_ids) 194 ] 195 self._run_workers("init_worker", all_kwargs=init_worker_all_kwargs) --> 197 self._run_workers("init_device") 198 self._run_workers("load_model", 199 max_concurrent_workers=self.parallel_config. 200 max_parallel_loading_workers)
File ~/lalb/vllm/vllm/executor/ray_tpu_executor.py:264, in RayTPUExecutor._run_workers(self, method, async_run_remote_workers_only, all_args, all_kwargs, use_dummy_driver, max_concurrent_workers, use_ray_compiled_dag, *args, **kwargs) 262 # Start the driver worker after all the ray workers. 263 if not use_dummy_driver: --> 264 driver_worker_output = self.driver_worker.execute_method( 265 method, *driver_args, **driver_kwargs) 266 else: 267 assert self.driver_dummy_worker is not None
File ~/lalb/vllm/vllm/worker/worker_base.py:465, in WorkerWrapperBase.execute_method(self, method, *args, **kwargs) 462 msg = (f"Error executing method {method}. " 463 "This might cause deadlock in distributed execution.") 464 logger.exception(msg) --> 465 raise e
File ~/lalb/vllm/vllm/worker/worker_base.py:456, in WorkerWrapperBase.execute_method(self, method, *args, **kwargs) 454 target = self if self.worker is None else self.worker 455 executor = getattr(target, method) --> 456 return executor(*args, **kwargs) 457 except Exception as e: 458 # if the driver worker also execute methods, 459 # exceptions in the rest worker may cause deadlock in rpc like ray 460 # see https://github.com/vllm-project/vllm/issues/3455 461 # print the error and inform the user to solve the error 462 msg = (f"Error executing method {method}. " 463 "This might cause deadlock in distributed execution.")
File ~/lalb/vllm/vllm/worker/tpu_worker.py:83, in TPUWorker.init_device(self) 72 # NOTE(woosuk): This is just to initialize the TP group and broadcast 73 # the input objects on CPU. The all-reduce and all-gather ops on TPU 74 # are invoked by
xm.all_reduceandxm.all_gatherwhich use their 75 # own context. 76 init_distributed_environment( 77 world_size=self.parallel_config.world_size, 78 rank=self.rank, (...) 81 backend="gloo", 82 ) ---> 83 ensure_model_parallel_initialized( 84 self.parallel_config.tensor_parallel_size, 85 self.parallel_config.pipeline_parallel_size) 87 # Device initialization should happen after initializing the distributed 88 # runtime. 89 self.device = xm.xla_device()File ~/lalb/vllm/vllm/distributed/parallel_state.py:1059, in ensure_model_parallel_initialized(tensor_model_parallel_size, pipeline_model_parallel_size, backend) 1056 backend = backend or torch.distributed.get_backend( 1057 get_world_group().device_group) 1058 if not model_parallel_is_initialized(): -> 1059 initialize_model_parallel(tensor_model_parallel_size, 1060 pipeline_model_parallel_size, backend) 1061 return 1063 assert ( 1064 get_tensor_model_parallel_world_size() == tensor_model_parallel_size 1065 ), ("tensor parallel group already initialized, but of unexpected size: " 1066 f"{get_tensor_model_parallel_world_size()=} vs. " 1067 f"{tensor_model_parallel_size=}")
File ~/lalb/vllm/vllm/distributed/parallel_state.py:1023, in initialize_model_parallel(tensor_model_parallel_size, pipeline_model_parallel_size, backend) 1020 group_ranks.append(ranks) 1022 # message queue broadcaster is only used in tensor model parallel group -> 1023 _TP = init_model_parallel_group(group_ranks, 1024 get_world_group().local_rank, 1025 backend, 1026 use_message_queue_broadcaster=True, 1027 group_name="tp") 1029 # Build the pipeline model-parallel groups. 1030 num_pipeline_model_parallel_groups: int = (world_size // 1031 pipeline_model_parallel_size)
File ~/lalb/vllm/vllm/distributed/parallel_state.py:864, in init_model_parallel_group(group_ranks, local_rank, backend, use_custom_allreduce, use_message_queue_broadcaster, group_name) 862 if use_custom_allreduce is None: 863 use_custom_allreduce = _ENABLE_CUSTOM_ALL_REDUCE --> 864 return GroupCoordinator( 865 group_ranks=group_ranks, 866 local_rank=local_rank, 867 torch_distributed_backend=backend, 868 use_pynccl=True, 869 use_custom_allreduce=use_custom_allreduce, 870 use_tpu_communicator=True, 871 use_message_queue_broadcaster=use_message_queue_broadcaster, 872 group_name=group_name, 873 )
File ~/lalb/vllm/vllm/distributed/parallel_state.py:231, in GroupCoordinator.init(self, group_ranks, local_rank, torch_distributed_backend, use_pynccl, use_custom_allreduce, use_tpu_communicator, use_message_queue_broadcaster, group_name) 229 self.tpu_communicator: Optional[TpuCommunicator] = None 230 if use_tpu_communicator and self.world_size > 1: --> 231 self.tpu_communicator = TpuCommunicator(group=self.cpu_group) 233 from vllm.distributed.device_communicators.shm_broadcast import ( 234 MessageQueue) 235 self.mq_broadcaster: Optional[MessageQueue] = None
File ~/lalb/vllm/vllm/distributed/device_communicators/tpu_communicator.py:53, in TpuCommunicator.init(self, group) 50 os.environ["CLOUD_TPU_TASK_ID"] = str(global_rank) 51 os.environ["TPU_VISIBLE_CHIPS"] = str(local_rank) ---> 53 pjrt.initialize_multiprocess(local_rank, local_world_size) 54 xr._init_world_size_ordinal()
File ~/.local/lib/python3.10/site-packages/torch_xla/_internal/pjrt.py:121, in initialize_multiprocess(local_rank, local_world_size) 118 elif runtime.device_type() == 'NEURON': 119 neuron.initialize_env(local_rank, local_world_size) --> 121 devices = xm.get_xla_supported_devices() 122 xm.set_replication(xm.xla_device(), devices)
File ~/.local/lib/python3.10/site-packages/torch_xla/core/xla_model.py:93, in get_xla_supported_devices(devkind, max_devices) 90 # TODO(wcromar): Remove
devkindafter 2.3 release cut. We no longer support 91 # multiple device types. 92 if not devkind: ---> 93 devices = torch_xla._XLAC._xla_get_devices() 94 return [ 95 f'xla:{i}' 96 for i, _ in enumerate(devices[:max_devices] if max_devices else devices) 97 ] 98 else:RuntimeError: Bad StatusOr access: UNKNOWN: TPU initialization failed: Invalid --2a886c8_slice_builder_worker_addresses specified. Expected 16 worker addresses, got 4. 2024-09-24 04:16:22,543 ERROR worker.py:409 -- Unhandled error (suppress with 'RAY_IGNORE_UNHANDLED_ERRORS=1'): ray::RayWorkerWrapper.execute_method() (pid=102895, ip=10.130.0.29, actor_id=7f8628802a0597f532e47c4d05000000, repr=<vllm.executor.ray_utils.RayWorkerWrapper object at 0x7efdc6b54370>) File "/home/spars/lalb/vllm/vllm/worker/worker_base.py", line 465, in execute_method raise e File "/home/spars/lalb/vllm/vllm/worker/worker_base.py", line 456, in execute_method return executor(*args, **kwargs) File "/home/spars/lalb/vllm/vllm/worker/tpu_worker.py", line 83, in init_device ensure_model_parallel_initialized( File "/home/spars/lalb/vllm/vllm/distributed/parallel_state.py", line 1059, in ensure_model_parallel_initialized initialize_model_parallel(tensor_model_parallel_size, File "/home/spars/lalb/vllm/vllm/distributed/parallel_state.py", line 1023, in initialize_model_parallel _TP = init_model_parallel_group(group_ranks, File "/home/spars/lalb/vllm/vllm/distributed/parallel_state.py", line 864, in init_model_parallel_group return GroupCoordinator( File "/home/spars/lalb/vllm/vllm/distributed/parallel_state.py", line 231, in init self.tpu_communicator = TpuCommunicator(group=self.cpu_group) File "/home/spars/lalb/vllm/vllm/distributed/device_communicators/tpu_communicator.py", line 53, in init pjrt.initialize_multiprocess(local_rank, local_world_size) File "/home/spars/.local/lib/python3.10/site-packages/torch_xla/_internal/pjrt.py", line 121, in initialize_multiprocess devices = xm.get_xla_supported_devices() File "/home/spars/.local/lib/python3.10/site-packages/torch_xla/core/xla_model.py", line 93, in get_xla_supported_devices devices = torch_xla._XLAC._xla_get_devices() RuntimeError: Bad StatusOr access: UNKNOWN: TPU initialization failed: Invalid --2a886c8_slice_builder_worker_addresses specified. Expected 16 worker addresses, got 4. 2024-09-24 04:16:22,545 ERROR worker.py:409 -- Unhandled error (suppress with 'RAY_IGNORE_UNHANDLED_ERRORS=1'): ray::RayWorkerWrapper.execute_method() (pid=102242, ip=10.130.0.28, actor_id=f9be6c3c131a5388e36c675f05000000, repr=<vllm.executor.ray_utils.RayWorkerWrapper object at 0x7f8cca7d4340>) File "/home/spars/lalb/vllm/vllm/worker/worker_base.py", line 465, in execute_method raise e File "/home/spars/lalb/vllm/vllm/worker/worker_base.py", line 456, in execute_method return executor(*args, **kwargs) File "/home/spars/lalb/vllm/vllm/worker/tpu_worker.py", line 83, in init_device ensure_model_parallel_initialized( File "/home/spars/lalb/vllm/vllm/distributed/parallel_state.py", line 1059, in ensure_model_parallel_initialized initialize_model_parallel(tensor_model_parallel_size, File "/home/spars/lalb/vllm/vllm/distributed/parallel_state.py", line 1023, in initialize_model_parallel _TP = init_model_parallel_group(group_ranks, File "/home/spars/lalb/vllm/vllm/distributed/parallel_state.py", line 864, in init_model_parallel_group return GroupCoordinator( File "/home/spars/lalb/vllm/vllm/distributed/parallel_state.py", line 231, in init self.tpu_communicator = TpuCommunicator(group=self.cpu_group) File "/home/spars/lalb/vllm/vllm/distributed/device_communicators/tpu_communicator.py", line 53, in init pjrt.initialize_multiprocess(local_rank, local_world_size) File "/home/spars/.local/lib/python3.10/site-packages/torch_xla/_internal/pjrt.py", line 121, in initialize_multiprocess devices = xm.get_xla_supported_devices() File "/home/spars/.local/lib/python3.10/site-packages/torch_xla/core/xla_model.py", line 93, in get_xla_supported_devices devices = torch_xla._XLAC._xla_get_devices() RuntimeError: Bad StatusOr access: UNKNOWN: TPU initialization failed: Invalid --2a886c8_slice_builder_worker_addresses specified. Expected 16 worker addresses, got 4. 2024-09-24 04:16:22,545 ERROR worker.py:409 -- Unhandled error (suppress with 'RAY_IGNORE_UNHANDLED_ERRORS=1'): ray::RayWorkerWrapper.execute_method() (pid=107836, ip=10.130.0.31, actor_id=36bfc0b00d2f6b8f6e679d4605000000, repr=<vllm.executor.ray_utils.RayWorkerWrapper object at 0x7fc7eadb43a0>) File "/home/spars/lalb/vllm/vllm/worker/worker_base.py", line 465, in execute_method raise e File "/home/spars/lalb/vllm/vllm/worker/worker_base.py", line 456, in execute_method return executor(*args, **kwargs) File "/home/spars/lalb/vllm/vllm/worker/tpu_worker.py", line 83, in init_device ensure_model_parallel_initialized( File "/home/spars/lalb/vllm/vllm/distributed/parallel_state.py", line 1059, in ensure_model_parallel_initialized initialize_model_parallel(tensor_model_parallel_size, File "/home/spars/lalb/vllm/vllm/distributed/parallel_state.py", line 1023, in initialize_model_parallel _TP = init_model_parallel_group(group_ranks, File "/home/spars/lalb/vllm/vllm/distributed/parallel_state.py", line 864, in init_model_parallel_group return GroupCoordinator( File "/home/spars/lalb/vllm/vllm/distributed/parallel_state.py", line 231, in init self.tpu_communicator = TpuCommunicator(group=self.cpu_group) File "/home/spars/lalb/vllm/vllm/distributed/device_communicators/tpu_communicator.py", line 53, in init pjrt.initialize_multiprocess(local_rank, local_world_size) File "/home/spars/.local/lib/python3.10/site-packages/torch_xla/_internal/pjrt.py", line 121, in initialize_multiprocess devices = xm.get_xla_supported_devices() File "/home/spars/.local/lib/python3.10/site-packages/torch_xla/core/xla_model.py", line 93, in get_xla_supported_devices devices = torch_xla._XLAC._xla_get_devices() RuntimeError: Bad StatusOr access: UNKNOWN: TPU initialization failed: Invalid --2a886c8_slice_builder_worker_addresses specified. Expected 16 worker addresses, got 4. 2024-09-24 04:16:22,546 ERROR worker.py:409 -- Unhandled error (suppress with 'RAY_IGNORE_UNHANDLED_ERRORS=1'): ray::RayWorkerWrapper.execute_method() (pid=107954, ip=10.130.0.31, actor_id=981d5cf72d8e3e73ae805ad805000000, repr=<vllm.executor.ray_utils.RayWorkerWrapper object at 0x7f19ddfe0460>) File "/home/spars/lalb/vllm/vllm/worker/worker_base.py", line 465, in execute_method raise e File "/home/spars/lalb/vllm/vllm/worker/worker_base.py", line 456, in execute_method return executor(*args, **kwargs) File "/home/spars/lalb/vllm/vllm/worker/tpu_worker.py", line 83, in init_device ensure_model_parallel_initialized( File "/home/spars/lalb/vllm/vllm/distributed/parallel_state.py", line 1059, in ensure_model_parallel_initialized initialize_model_parallel(tensor_model_parallel_size, File "/home/spars/lalb/vllm/vllm/distributed/parallel_state.py", line 1023, in initialize_model_parallel _TP = init_model_parallel_group(group_ranks, File "/home/spars/lalb/vllm/vllm/distributed/parallel_state.py", line 864, in init_model_parallel_group return GroupCoordinator( File "/home/spars/lalb/vllm/vllm/distributed/parallel_state.py", line 231, in init self.tpu_communicator = TpuCommunicator(group=self.cpu_group) File "/home/spars/lalb/vllm/vllm/distributed/device_communicators/tpu_communicator.py", line 53, in init pjrt.initialize_multiprocess(local_rank, local_world_size) File "/home/spars/.local/lib/python3.10/site-packages/torch_xla/_internal/pjrt.py", line 121, in initialize_multiprocess devices = xm.get_xla_supported_devices() File "/home/spars/.local/lib/python3.10/site-packages/torch_xla/core/xla_model.py", line 93, in get_xla_supported_devices devices = torch_xla._XLAC._xla_get_devices() RuntimeError: Bad StatusOr access: UNKNOWN: TPU initialization failed: Invalid --2a886c8_slice_builder_worker_addresses specified. Expected 16 worker addresses, got 4. 2024-09-24 04:16:22,547 ERROR worker.py:409 -- Unhandled error (suppress with 'RAY_IGNORE_UNHANDLED_ERRORS=1'): ray::RayWorkerWrapper.execute_method() (pid=108195, ip=10.130.0.31, actor_id=d95938ac69d962262223027105000000, repr=<vllm.executor.ray_utils.RayWorkerWrapper object at 0x7f47518c8460>) File "/home/spars/lalb/vllm/vllm/worker/worker_base.py", line 465, in execute_method raise e File "/home/spars/lalb/vllm/vllm/worker/worker_base.py", line 456, in execute_method return executor(*args, **kwargs) File "/home/spars/lalb/vllm/vllm/worker/tpu_worker.py", line 83, in init_device ensure_model_parallel_initialized( File "/home/spars/lalb/vllm/vllm/distributed/parallel_state.py", line 1059, in ensure_model_parallel_initialized initialize_model_parallel(tensor_model_parallel_size, File "/home/spars/lalb/vllm/vllm/distributed/parallel_state.py", line 1023, in initialize_model_parallel _TP = init_model_parallel_group(group_ranks, File "/home/spars/lalb/vllm/vllm/distributed/parallel_state.py", line 864, in init_model_parallel_group return GroupCoordinator( File "/home/spars/lalb/vllm/vllm/distributed/parallel_state.py", line 231, in init self.tpu_communicator = TpuCommunicator(group=self.cpu_group) File "/home/spars/lalb/vllm/vllm/distributed/device_communicators/tpu_communicator.py", line 53, in init pjrt.initialize_multiprocess(local_rank, local_world_size) File "/home/spars/.local/lib/python3.10/site-packages/torch_xla/_internal/pjrt.py", line 121, in initialize_multiprocess devices = xm.get_xla_supported_devices() File "/home/spars/.local/lib/python3.10/site-packages/torch_xla/core/xla_model.py", line 93, in get_xla_supported_devices devices = torch_xla._XLAC._xla_get_devices() RuntimeError: Bad StatusOr access: UNKNOWN: TPU initialization failed: Invalid --2a886c8_slice_builder_worker_addresses specified. Expected 16 worker addresses, got 4. 2024-09-24 04:16:22,547 ERROR worker.py:409 -- Unhandled error (suppress with 'RAY_IGNORE_UNHANDLED_ERRORS=1'): ray::RayWorkerWrapper.execute_method() (pid=358493, ip=10.130.0.30, actor_id=cdceaff2c60b22f8a44b6f3a05000000, repr=<vllm.executor.ray_utils.RayWorkerWrapper object at 0x7f106e20cc70>) File "/home/spars/lalb/vllm/vllm/worker/worker_base.py", line 465, in execute_method raise e File "/home/spars/lalb/vllm/vllm/worker/worker_base.py", line 456, in execute_method return executor(*args, **kwargs) File "/home/spars/lalb/vllm/vllm/worker/tpu_worker.py", line 83, in init_device ensure_model_parallel_initialized( File "/home/spars/lalb/vllm/vllm/distributed/parallel_state.py", line 1059, in ensure_model_parallel_initialized initialize_model_parallel(tensor_model_parallel_size, File "/home/spars/lalb/vllm/vllm/distributed/parallel_state.py", line 1023, in initialize_model_parallel _TP = init_model_parallel_group(group_ranks, File "/home/spars/lalb/vllm/vllm/distributed/parallel_state.py", line 864, in init_model_parallel_group return GroupCoordinator( File "/home/spars/lalb/vllm/vllm/distributed/parallel_state.py", line 231, in init self.tpu_communicator = TpuCommunicator(group=self.cpu_group) File "/home/spars/lalb/vllm/vllm/distributed/device_communicators/tpu_communicator.py", line 53, in init pjrt.initialize_multiprocess(local_rank, local_world_size) File "/home/spars/.local/lib/python3.10/site-packages/torch_xla/_internal/pjrt.py", line 121, in initialize_multiprocess devices = xm.get_xla_supported_devices() File "/home/spars/.local/lib/python3.10/site-packages/torch_xla/core/xla_model.py", line 93, in get_xla_supported_devices devices = torch_xla._XLAC._xla_get_devices() RuntimeError: Bad StatusOr access: UNKNOWN: TPU initialization failed: Invalid --2a886c8_slice_builder_worker_addresses specified. Expected 16 worker addresses, got 4. 2024-09-24 04:16:22,548 ERROR worker.py:409 -- Unhandled error (suppress with 'RAY_IGNORE_UNHANDLED_ERRORS=1'): ray::RayWorkerWrapper.execute_method() (pid=358251, ip=10.130.0.30, actor_id=bf0d12344c31f6814e6ba14905000000, repr=<vllm.executor.ray_utils.RayWorkerWrapper object at 0x7facc7f0cb50>) File "/home/spars/lalb/vllm/vllm/worker/worker_base.py", line 465, in execute_method raise e File "/home/spars/lalb/vllm/vllm/worker/worker_base.py", line 456, in execute_method return executor(*args, **kwargs) File "/home/spars/lalb/vllm/vllm/worker/tpu_worker.py", line 83, in init_device ensure_model_parallel_initialized( File "/home/spars/lalb/vllm/vllm/distributed/parallel_state.py", line 1059, in ensure_model_parallel_initialized initialize_model_parallel(tensor_model_parallel_size, File "/home/spars/lalb/vllm/vllm/distributed/parallel_state.py", line 1023, in initialize_model_parallel _TP = init_model_parallel_group(group_ranks, File "/home/spars/lalb/vllm/vllm/distributed/parallel_state.py", line 864, in init_model_parallel_group return GroupCoordinator( File "/home/spars/lalb/vllm/vllm/distributed/parallel_state.py", line 231, in init self.tpu_communicator = TpuCommunicator(group=self.cpu_group) File "/home/spars/lalb/vllm/vllm/distributed/device_communicators/tpu_communicator.py", line 53, in init pjrt.initialize_multiprocess(local_rank, local_world_size) File "/home/spars/.local/lib/python3.10/site-packages/torch_xla/_internal/pjrt.py", line 121, in initialize_multiprocess devices = xm.get_xla_supported_devices() File "/home/spars/.local/lib/python3.10/site-packages/torch_xla/core/xla_model.py", line 93, in get_xla_supported_devices devices = torch_xla._XLAC._xla_get_devices() RuntimeError: Bad StatusOr access: UNKNOWN: TPU initialization failed: Invalid --2a886c8_slice_builder_worker_addresses specified. Expected 16 worker addresses, got 4. 2024-09-24 04:16:22,549 ERROR worker.py:409 -- Unhandled error (suppress with 'RAY_IGNORE_UNHANDLED_ERRORS=1'): ray::RayWorkerWrapper.execute_method() (pid=102774, ip=10.130.0.29, actor_id=b0b2dfe4b5573a904c7c9afa05000000, repr=<vllm.executor.ray_utils.RayWorkerWrapper object at 0x7f6a2cd9c370>) File "/home/spars/lalb/vllm/vllm/worker/worker_base.py", line 465, in execute_method raise e File "/home/spars/lalb/vllm/vllm/worker/worker_base.py", line 456, in execute_method return executor(*args, **kwargs) File "/home/spars/lalb/vllm/vllm/worker/tpu_worker.py", line 83, in init_device ensure_model_parallel_initialized( File "/home/spars/lalb/vllm/vllm/distributed/parallel_state.py", line 1059, in ensure_model_parallel_initialized initialize_model_parallel(tensor_model_parallel_size, File "/home/spars/lalb/vllm/vllm/distributed/parallel_state.py", line 1023, in initialize_model_parallel _TP = init_model_parallel_group(group_ranks, File "/home/spars/lalb/vllm/vllm/distributed/parallel_state.py", line 864, in init_model_parallel_group return GroupCoordinator( File "/home/spars/lalb/vllm/vllm/distributed/parallel_state.py", line 231, in init self.tpu_communicator = TpuCommunicator(group=self.cpu_group) File "/home/spars/lalb/vllm/vllm/distributed/device_communicators/tpu_communicator.py", line 53, in init pjrt.initialize_multiprocess(local_rank, local_world_size) File "/home/spars/.local/lib/python3.10/site-packages/torch_xla/_internal/pjrt.py", line 121, in initialize_multiprocess devices = xm.get_xla_supported_devices() File "/home/spars/.local/lib/python3.10/site-packages/torch_xla/core/xla_model.py", line 93, in get_xla_supported_devices devices = torch_xla._XLAC._xla_get_devices() RuntimeError: Bad StatusOr access: UNKNOWN: TPU initialization failed: Invalid --2a886c8_slice_builder_worker_addresses specified. Expected 16 worker addresses, got 4. 2024-09-24 04:16:22,549 ERROR worker.py:409 -- Unhandled error (suppress with 'RAY_IGNORE_UNHANDLED_ERRORS=1'): ray::RayWorkerWrapper.execute_method() (pid=102642, ip=10.130.0.29, actor_id=aff0082536fdfa98501960d105000000, repr=<vllm.executor.ray_utils.RayWorkerWrapper object at 0x7ef3fdf5c3d0>) File "/home/spars/lalb/vllm/vllm/worker/worker_base.py", line 465, in execute_method raise e File "/home/spars/lalb/vllm/vllm/worker/worker_base.py", line 456, in execute_method return executor(*args, **kwargs) File "/home/spars/lalb/vllm/vllm/worker/tpu_worker.py", line 83, in init_device ensure_model_parallel_initialized( File "/home/spars/lalb/vllm/vllm/distributed/parallel_state.py", line 1059, in ensure_model_parallel_initialized initialize_model_parallel(tensor_model_parallel_size, File "/home/spars/lalb/vllm/vllm/distributed/parallel_state.py", line 1023, in initialize_model_parallel _TP = init_model_parallel_group(group_ranks, File "/home/spars/lalb/vllm/vllm/distributed/parallel_state.py", line 864, in init_model_parallel_group return GroupCoordinator( File "/home/spars/lalb/vllm/vllm/distributed/parallel_state.py", line 231, in init self.tpu_communicator = TpuCommunicator(group=self.cpu_group) File "/home/spars/lalb/vllm/vllm/distributed/device_communicators/tpu_communicator.py", line 53, in init pjrt.initialize_multiprocess(local_rank, local_world_size) File "/home/spars/.local/lib/python3.10/site-packages/torch_xla/_internal/pjrt.py", line 121, in initialize_multiprocess devices = xm.get_xla_supported_devices() File "/home/spars/.local/lib/python3.10/site-packages/torch_xla/core/xla_model.py", line 93, in get_xla_supported_devices devices = torch_xla._XLAC._xla_get_devices() RuntimeError: Bad StatusOr access: UNKNOWN: TPU initialization failed: Invalid --2a886c8_slice_builder_worker_addresses specified. Expected 16 worker addresses, got 4. 2024-09-24 04:16:22,550 ERROR worker.py:409 -- Unhandled error (suppress with 'RAY_IGNORE_UNHANDLED_ERRORS=1'): ray::RayWorkerWrapper.execute_method() (pid=358372, ip=10.130.0.30, actor_id=5bc8014451c573642bead5b805000000, repr=<vllm.executor.ray_utils.RayWorkerWrapper object at 0x7f5a7f56cc40>) File "/home/spars/lalb/vllm/vllm/worker/worker_base.py", line 465, in execute_method raise e File "/home/spars/lalb/vllm/vllm/worker/worker_base.py", line 456, in execute_method return executor(*args, **kwargs) File "/home/spars/lalb/vllm/vllm/worker/tpu_worker.py", line 83, in init_device ensure_model_parallel_initialized( File "/home/spars/lalb/vllm/vllm/distributed/parallel_state.py", line 1059, in ensure_model_parallel_initialized initialize_model_parallel(tensor_model_parallel_size, File "/home/spars/lalb/vllm/vllm/distributed/parallel_state.py", line 1023, in initialize_model_parallel _TP = init_model_parallel_group(group_ranks, File "/home/spars/lalb/vllm/vllm/distributed/parallel_state.py", line 864, in init_model_parallel_group return GroupCoordinator( File "/home/spars/lalb/vllm/vllm/distributed/parallel_state.py", line 231, in init self.tpu_communicator = TpuCommunicator(group=self.cpu_group) File "/home/spars/lalb/vllm/vllm/distributed/device_communicators/tpu_communicator.py", line 53, in init pjrt.initialize_multiprocess(local_rank, local_world_size) File "/home/spars/.local/lib/python3.10/site-packages/torch_xla/_internal/pjrt.py", line 121, in initialize_multiprocess devices = xm.get_xla_supported_devices() File "/home/spars/.local/lib/python3.10/site-packages/torch_xla/core/xla_model.py", line 93, in get_xla_supported_devices devices = torch_xla._XLAC._xla_get_devices() RuntimeError: Bad StatusOr access: UNKNOWN: TPU initialization failed: Invalid --2a886c8_slice_builder_worker_addresses specified. Expected 16 worker addresses, got 4. 2024-09-24 04:16:22,551 ERROR worker.py:409 -- Unhandled error (suppress with 'RAY_IGNORE_UNHANDLED_ERRORS=1'): ray::RayWorkerWrapper.execute_method() (pid=102124, ip=10.130.0.28, actor_id=79b5099eb5a9274e1b9b75f405000000, repr=<vllm.executor.ray_utils.RayWorkerWrapper object at 0x7f2851dc83d0>) File "/home/spars/lalb/vllm/vllm/worker/worker_base.py", line 465, in execute_method raise e File "/home/spars/lalb/vllm/vllm/worker/worker_base.py", line 456, in execute_method return executor(*args, **kwargs) File "/home/spars/lalb/vllm/vllm/worker/tpu_worker.py", line 83, in init_device ensure_model_parallel_initialized( File "/home/spars/lalb/vllm/vllm/distributed/parallel_state.py", line 1059, in ensure_model_parallel_initialized initialize_model_parallel(tensor_model_parallel_size, File "/home/spars/lalb/vllm/vllm/distributed/parallel_state.py", line 1023, in initialize_model_parallel _TP = init_model_parallel_group(group_ranks, File "/home/spars/lalb/vllm/vllm/distributed/parallel_state.py", line 864, in init_model_parallel_group return GroupCoordinator( File "/home/spars/lalb/vllm/vllm/distributed/parallel_state.py", line 231, in init self.tpu_communicator = TpuCommunicator(group=self.cpu_group) File "/home/spars/lalb/vllm/vllm/distributed/device_communicators/tpu_communicator.py", line 53, in init pjrt.initialize_multiprocess(local_rank, local_world_size) File "/home/spars/.local/lib/python3.10/site-packages/torch_xla/_internal/pjrt.py", line 121, in initialize_multiprocess devices = xm.get_xla_supported_devices() File "/home/spars/.local/lib/python3.10/site-packages/torch_xla/core/xla_model.py", line 93, in get_xla_supported_devices devices = torch_xla._XLAC._xla_get_devices() RuntimeError: Bad StatusOr access: UNKNOWN: TPU initialization failed: Invalid --2a886c8_slice_builder_worker_addresses specified. Expected 16 worker addresses, got 4. 2024-09-24 04:16:22,551 ERROR worker.py:409 -- Unhandled error (suppress with 'RAY_IGNORE_UNHANDLED_ERRORS=1'): ray::RayWorkerWrapper.execute_method() (pid=108075, ip=10.130.0.31, actor_id=4cb98bf15cd4d28c339203ff05000000, repr=<vllm.executor.ray_utils.RayWorkerWrapper object at 0x7ef72e2f0430>) File "/home/spars/lalb/vllm/vllm/worker/worker_base.py", line 465, in execute_method raise e File "/home/spars/lalb/vllm/vllm/worker/worker_base.py", line 456, in execute_method return executor(*args, **kwargs) File "/home/spars/lalb/vllm/vllm/worker/tpu_worker.py", line 83, in init_device ensure_model_parallel_initialized( File "/home/spars/lalb/vllm/vllm/distributed/parallel_state.py", line 1059, in ensure_model_parallel_initialized initialize_model_parallel(tensor_model_parallel_size, File "/home/spars/lalb/vllm/vllm/distributed/parallel_state.py", line 1023, in initialize_model_parallel _TP = init_model_parallel_group(group_ranks, File "/home/spars/lalb/vllm/vllm/distributed/parallel_state.py", line 864, in init_model_parallel_group return GroupCoordinator( File "/home/spars/lalb/vllm/vllm/distributed/parallel_state.py", line 231, in init self.tpu_communicator = TpuCommunicator(group=self.cpu_group) File "/home/spars/lalb/vllm/vllm/distributed/device_communicators/tpu_communicator.py", line 53, in init pjrt.initialize_multiprocess(local_rank, local_world_size) File "/home/spars/.local/lib/python3.10/site-packages/torch_xla/_internal/pjrt.py", line 121, in initialize_multiprocess devices = xm.get_xla_supported_devices() File "/home/spars/.local/lib/python3.10/site-packages/torch_xla/core/xla_model.py", line 93, in get_xla_supported_devices devices = torch_xla._XLAC._xla_get_devices() RuntimeError: Bad StatusOr access: UNKNOWN: TPU initialization failed: Invalid --2a886c8_slice_builder_worker_addresses specified. Expected 16 worker addresses, got 4. 2024-09-24 04:16:22,552 ERROR worker.py:409 -- Unhandled error (suppress with 'RAY_IGNORE_UNHANDLED_ERRORS=1'): ray::RayWorkerWrapper.execute_method() (pid=102480, ip=10.130.0.28, actor_id=5a63b3a07dfd9368f3d8145305000000, repr=<vllm.executor.ray_utils.RayWorkerWrapper object at 0x7f1fe20e0370>) File "/home/spars/lalb/vllm/vllm/worker/worker_base.py", line 465, in execute_method raise e File "/home/spars/lalb/vllm/vllm/worker/worker_base.py", line 456, in execute_method return executor(*args, **kwargs) File "/home/spars/lalb/vllm/vllm/worker/tpu_worker.py", line 83, in init_device ensure_model_parallel_initialized( File "/home/spars/lalb/vllm/vllm/distributed/parallel_state.py", line 1059, in ensure_model_parallel_initialized initialize_model_parallel(tensor_model_parallel_size, File "/home/spars/lalb/vllm/vllm/distributed/parallel_state.py", line 1023, in initialize_model_parallel _TP = init_model_parallel_group(group_ranks, File "/home/spars/lalb/vllm/vllm/distributed/parallel_state.py", line 864, in init_model_parallel_group return GroupCoordinator( File "/home/spars/lalb/vllm/vllm/distributed/parallel_state.py", line 231, in init self.tpu_communicator = TpuCommunicator(group=self.cpu_group) File "/home/spars/lalb/vllm/vllm/distributed/device_communicators/tpu_communicator.py", line 53, in init pjrt.initialize_multiprocess(local_rank, local_world_size) File "/home/spars/.local/lib/python3.10/site-packages/torch_xla/_internal/pjrt.py", line 121, in initialize_multiprocess devices = xm.get_xla_supported_devices() File "/home/spars/.local/lib/python3.10/site-packages/torch_xla/core/xla_model.py", line 93, in get_xla_supported_devices devices = torch_xla._XLAC._xla_get_devices() RuntimeError: Bad StatusOr access: UNKNOWN: TPU initialization failed: Invalid --2a886c8_slice_builder_worker_addresses specified. Expected 16 worker addresses, got 4. 2024-09-24 04:16:22,553 ERROR worker.py:409 -- Unhandled error (suppress with 'RAY_IGNORE_UNHANDLED_ERRORS=1'): ray::RayWorkerWrapper.execute_method() (pid=103014, ip=10.130.0.29, actor_id=2229b24714284766de33cfbc05000000, repr=<vllm.executor.ray_utils.RayWorkerWrapper object at 0x7f995c5ec400>) File "/home/spars/lalb/vllm/vllm/worker/worker_base.py", line 465, in execute_method raise e File "/home/spars/lalb/vllm/vllm/worker/worker_base.py", line 456, in execute_method return executor(*args, **kwargs) File "/home/spars/lalb/vllm/vllm/worker/tpu_worker.py", line 83, in init_device ensure_model_parallel_initialized( File "/home/spars/lalb/vllm/vllm/distributed/parallel_state.py", line 1059, in ensure_model_parallel_initialized initialize_model_parallel(tensor_model_parallel_size, File "/home/spars/lalb/vllm/vllm/distributed/parallel_state.py", line 1023, in initialize_model_parallel _TP = init_model_parallel_group(group_ranks, File "/home/spars/lalb/vllm/vllm/distributed/parallel_state.py", line 864, in init_model_parallel_group return GroupCoordinator( File "/home/spars/lalb/vllm/vllm/distributed/parallel_state.py", line 231, in init self.tpu_communicator = TpuCommunicator(group=self.cpu_group) File "/home/spars/lalb/vllm/vllm/distributed/device_communicators/tpu_communicator.py", line 53, in init pjrt.initialize_multiprocess(local_rank, local_world_size) File "/home/spars/.local/lib/python3.10/site-packages/torch_xla/_internal/pjrt.py", line 121, in initialize_multiprocess devices = xm.get_xla_supported_devices() File "/home/spars/.local/lib/python3.10/site-packages/torch_xla/core/xla_model.py", line 93, in get_xla_supported_devices devices = torch_xla._XLAC._xla_get_devices() RuntimeError: Bad StatusOr access: UNKNOWN: TPU initialization failed: Invalid --2a886c8_slice_builder_worker_addresses specified. Expected 16 worker addresses, got 4. 2024-09-24 04:16:22,554 ERROR worker.py:409 -- Unhandled error (suppress with 'RAY_IGNORE_UNHANDLED_ERRORS=1'): ray::RayWorkerWrapper.execute_method() (pid=102361, ip=10.130.0.28, actor_id=e7639c923ffb04993a07e4cc05000000, repr=<vllm.executor.ray_utils.RayWorkerWrapper object at 0x7faf4b3203d0>) File "/home/spars/lalb/vllm/vllm/worker/worker_base.py", line 465, in execute_method raise e File "/home/spars/lalb/vllm/vllm/worker/worker_base.py", line 456, in execute_method return executor(*args, **kwargs) File "/home/spars/lalb/vllm/vllm/worker/tpu_worker.py", line 83, in init_device ensure_model_parallel_initialized( File "/home/spars/lalb/vllm/vllm/distributed/parallel_state.py", line 1059, in ensure_model_parallel_initialized initialize_model_parallel(tensor_model_parallel_size, File "/home/spars/lalb/vllm/vllm/distributed/parallel_state.py", line 1023, in initialize_model_parallel _TP = init_model_parallel_group(group_ranks, File "/home/spars/lalb/vllm/vllm/distributed/parallel_state.py", line 864, in init_model_parallel_group return GroupCoordinator( File "/home/spars/lalb/vllm/vllm/distributed/parallel_state.py", line 231, in init self.tpu_communicator = TpuCommunicator(group=self.cpu_group) File "/home/spars/lalb/vllm/vllm/distributed/device_communicators/tpu_communicator.py", line 53, in init pjrt.initialize_multiprocess(local_rank, local_world_size) File "/home/spars/.local/lib/python3.10/site-packages/torch_xla/_internal/pjrt.py", line 121, in initialize_multiprocess devices = xm.get_xla_supported_devices() File "/home/spars/.local/lib/python3.10/site-packages/torch_xla/core/xla_model.py", line 93, in get_xla_supported_devices devices = torch_xla._XLAC._xla_get_devices() RuntimeError: Bad StatusOr access: UNKNOWN: TPU initialization failed: Invalid --2a886c8_slice_builder_worker_addresses specified. Expected 16 worker addresses, got 4. tt = llm.get_tokenizer() I am running it on TPU V4-32 , with ray setup as this ======== Autoscaler status: 2024-09-24 04:19:17.599164 ======== Node status
Active: 1 node_a719383e498cb023fd7e8139291e66861cb40bd13862a676f52ac5e9 1 node_8415f9cdd51ddbb01cab862238db4f51e7c47fee37cc7eddb477e3d9 1 node_00327d1caf66f2ac5ed52dd91d724dc49662d7284afb6d90befa4e77 1 node_d6e3b308eb1f88faa69cb0c2bdf9749bacd3c51c79b37487e08d0a46 Pending: (no pending nodes) Recent failures: (no failures)
Resources
Usage: 0.0/960.0 CPU 16.0/16.0 TPU (16.0 used of 16.0 reserved in placement groups) 0.0/1.0 TPU-v4-32-head 0.0/4.0 fspar 0B/1.08TiB memory 0B/476.45GiB object_store_memory
Demands: (no resource demands)
- Assistant
The error you're encountering indicates a mismatch between the expected and provided TPU worker addresses during the initialization of your TPU setup with
vllmandRay. Here's a detailed breakdown and guidance on how to resolve this issue.Error Breakdown
-
Core Error Message:
RuntimeError: Bad StatusOr access: UNKNOWN: TPU initialization failed: Invalid --2a886c8_slice_builder_worker_addresses specified. Expected 16 worker addresses, got 4. -
Root Cause:
- Expectation vs. Reality: The TPU initialization process expects 16 worker addresses but only 4 are provided.
- Worker Addresses: These addresses are crucial for orchestrating parallel processing across TPU cores. A mismatch here prevents proper initialization and communication among TPU workers.
-
Contextual Information:
- TPU Type: You're using a TPU V4-32, which typically implies 32 TPU cores.
- Ray Setup: Your Ray autoscaler is configured with 4 active nodes, each potentially handling a subset of TPU cores.
Potential Causes
-
Configuration Mismatch:
- Expected Workers: The system is configured to expect 16 TPU workers.
- Provided Workers: Only 4 TPU workers are currently active or provided.
-
Resource Allocation Issues:
- Ray Placement Groups: The placement groups in Ray might be reserving fewer TPU resources than expected.
- Cluster Scaling: The Ray autoscaler might not have scaled up to the required number of nodes/workers.
-
Incorrect Environment Variables:
- There might be misconfigurations in environment variables related to TPU worker addresses.
Steps to Resolve
-
Verify Configuration Files:
- vllm Configuration: Check your
vllmconfiguration files to see if the number of TPU workers or slices is explicitly set to 16. - Ray Configuration: Ensure that the Ray cluster configuration aligns with the number of TPU workers you intend to use.
- vllm Configuration: Check your
-
Adjust Worker Expectations:
- If You Intend to Use 4 Workers:
- Update Configurations: Modify the configuration files to expect 4 worker addresses instead of 16.
- Environment Variables: Ensure that any environment variables or command-line arguments reflect the correct number of workers.
- If You Intend to Use 16 Workers:
- Scale Up Ray Cluster: Provision additional nodes in your Ray cluster to meet the requirement of 16 workers.
- Ensure Network Configuration: Verify that all 16 workers can communicate effectively, avoiding network issues that might prevent Ray from recognizing additional workers.
- If You Intend to Use 4 Workers:
-
Review Placement Groups:
- Ray Placement Groups: Ensure that the placement groups are correctly reserving the required TPU resources.
- Resource Allocation: Confirm that each node in the Ray cluster is correctly allocated the expected number of TPU cores.
-
Check TPU Initialization Parameters:
- Initialization Scripts: Review any scripts or commands that initialize the TPU workers to ensure they're correctly specifying the number of worker addresses.
- Logging: Enable detailed logging during TPU initialization to capture more granular error messages, which can provide further insights.
-
Consult Documentation & Support:
- vllm Documentation: Review the
vllmdocumentation to understand the expected configurations for TPU setups. - Ray Documentation: Ensure that your Ray setup follows best practices for scaling TPU resources.
- TPU Documentation: Refer to Google's TPU documentation for any version-specific configurations or limitations.
- vllm Documentation: Review the
Example Configuration Adjustment
If you determine that you should only have 4 workers, adjust your configuration accordingly. For instance:
# Example vllm configuration snippet parallel_config: tensor_parallel_size: 4 pipeline_parallel_size: 1 # Other configurations...Ensure that the placeholder or environment variable supplying
--2a886c8_slice_builder_worker_addressesaligns with having 4 worker addresses.Final Recommendations
- Consistent Configuration: Ensure that all parts of your system (vllm, Ray, TPU initialization scripts) consistently expect and provide the same number of TPU workers.
- Incremental Scaling: Start with a smaller number of workers (e.g., 4) and verify successful initialization before scaling up to 16.
- Monitor Resource Usage: Use Ray's dashboard and logging tools to monitor resource allocation and identify bottlenecks or mismatches early.
By aligning the expected number of TPU workers with the actual resources provided, you should be able to resolve the initialization errors and successfully run your
vllmtasks on the TPU V4-32 setup. -