Skip to content
TheChatArchiveThe Chat Archive
gpt-4o-2024-08-06· OpenAIAi2 WildChat-4.8M (AI2)

Что за ошибка { "name": "ValueError", "message": "You are trying to save a non…

  1. User

    Что за ошибка

    { "name": "ValueError", "message": "You are trying to save a non contiguous tensor: encoder.block.0.layer.0.SelfAttention.q.weight which is not allowed. It either means you are trying to save tensors which are reference of each other in which case it's recommended to save only the full tensors, and reslice at load time, or simply call .contiguous() on your tensor to pack it before saving.", "stack": "--------------------------------------------------------------------------- ValueError Traceback (most recent call last) Cell In[14], line 1 ----> 1 trainer.train()

    File ~/.local/lib/python3.10/site-packages/transformers/trainer.py:1938, in Trainer.train(self, resume_from_checkpoint, trial, ignore_keys_for_eval, **kwargs) 1936 hf_hub_utils.enable_progress_bars() 1937 else: -> 1938 return inner_training_loop( 1939 args=args, 1940 resume_from_checkpoint=resume_from_checkpoint, 1941 trial=trial, 1942 ignore_keys_for_eval=ignore_keys_for_eval, 1943 )

    File ~/.local/lib/python3.10/site-packages/transformers/trainer.py:2356, in Trainer._inner_training_loop(self, batch_size, args, resume_from_checkpoint, trial, ignore_keys_for_eval) 2353 self.state.epoch = epoch + (step + 1 + steps_skipped) / steps_in_epoch 2354 self.control = self.callback_handler.on_step_end(args, self.state, self.control) -> 2356 self._maybe_log_save_evaluate(tr_loss, grad_norm, model, trial, epoch, ignore_keys_for_eval) 2357 else: 2358 self.control = self.callback_handler.on_substep_end(args, self.state, self.control)

    File ~/.local/lib/python3.10/site-packages/transformers/trainer.py:2807, in Trainer._maybe_log_save_evaluate(self, tr_loss, grad_norm, model, trial, epoch, ignore_keys_for_eval) 2804 metrics = self._evaluate(trial, ignore_keys_for_eval) 2806 if self.control.should_save: -> 2807 self._save_checkpoint(model, trial, metrics=metrics) 2808 self.control = self.callback_handler.on_save(self.args, self.state, self.control)

    File ~/.local/lib/python3.10/site-packages/transformers/trainer.py:2886, in Trainer._save_checkpoint(self, model, trial, metrics) 2884 run_dir = self._get_output_dir(trial=trial) 2885 output_dir = os.path.join(run_dir, checkpoint_folder) -> 2886 self.save_model(output_dir, _internal_call=True) 2888 if not self.args.save_only_model: 2889 # Save optimizer and scheduler 2890 self._save_optimizer_and_scheduler(output_dir)

    File ~/.local/lib/python3.10/site-packages/transformers/trainer.py:3454, in Trainer.save_model(self, output_dir, _internal_call) 3451 self.model_wrapped.save_checkpoint(output_dir) 3453 elif self.args.should_save: -> 3454 self._save(output_dir) 3456 # Push to the Hub when save_model is called by the user. 3457 if self.args.push_to_hub and not _internal_call:

    File ~/.local/lib/python3.10/site-packages/transformers/trainer.py:3525, in Trainer._save(self, output_dir, state_dict) 3523 torch.save(state_dict, os.path.join(output_dir, WEIGHTS_NAME)) 3524 else: -> 3525 self.model.save_pretrained( 3526 output_dir, state_dict=state_dict, safe_serialization=self.args.save_safetensors 3527 ) 3529 if self.tokenizer is not None: 3530 self.tokenizer.save_pretrained(output_dir)

    File ~/.local/lib/python3.10/site-packages/transformers/modeling_utils.py:2793, in PreTrainedModel.save_pretrained(self, save_directory, is_main_process, state_dict, save_function, push_to_hub, max_shard_size, safe_serialization, variant, token, save_peft_format, **kwargs) 2788 gc.collect() 2790 if safe_serialization: 2791 # At some point we will need to deal better with save_function (used for TPU and other distributed 2792 # joyfulness), but for now this enough. -> 2793 safe_save_file(shard, os.path.join(save_directory, shard_file), metadata={"format": "pt"}) 2794 else: 2795 save_function(shard, os.path.join(save_directory, shard_file))

    File ~/.local/lib/python3.10/site-packages/safetensors/torch.py:281, in save_file(tensors, filename, metadata) 250 def save_file( 251 tensors: Dict[str, torch.Tensor], 252 filename: Union[str, os.PathLike], 253 metadata: Optional[Dict[str, str]] = None, 254 ): 255 """ 256 Saves a dictionary of tensors into raw bytes in safetensors format. 257 (...) 279 ``` 280 """ --> 281 serialize_file(_flatten(tensors), filename, metadata=metadata)

    File ~/.local/lib/python3.10/site-packages/safetensors/torch.py:485, in _flatten(tensors) 476 if failing: 477 raise RuntimeError( 478 f""" 479 Some tensors share memory, this will lead to duplicate memory on disk and potential differences when loading them again: {failing}. (...) 482 """ 483 ) --> 485 return { 486 k: { 487 "dtype": str(v.dtype).split(".")[-1], 488 "shape": v.shape, 489 "data": _tobytes(v, k), 490 } 491 for k, v in tensors.items() 492 }

    File ~/.local/lib/python3.10/site-packages/safetensors/torch.py:489, in (.0) 476 if failing: 477 raise RuntimeError( 478 f""" 479 Some tensors share memory, this will lead to duplicate memory on disk and potential differences when loading them again: {failing}. (...) 482 """ 483 ) 485 return { 486 k: { 487 "dtype": str(v.dtype).split(".")[-1], 488 "shape": v.shape, --> 489 "data": _tobytes(v, k), 490 } 491 for k, v in tensors.items() 492 }

    File ~/.local/lib/python3.10/site-packages/safetensors/torch.py:403, in _tobytes(tensor, name) 396 raise ValueError( 397 f"You are trying to save a sparse tensor: {name} which this library does not support." 398 " You can make it a dense tensor before saving with .to_dense() but be aware this might" 399 " make a much larger file than needed." 400 ) 402 if not tensor.is_contiguous(): --> 403 raise ValueError( 404 f"You are trying to save a non contiguous tensor: {name} which is not allowed. It either means you" 405 " are trying to save tensors which are reference of each other in which case it's recommended to save" 406 " only the full tensors, and reslice at load time, or simply call .contiguous() on your tensor to" 407 " pack it before saving." 408 ) 409 if tensor.device.type != "cpu": 410 # Moving tensor to cpu before saving 411 tensor = tensor.to("cpu")

    ValueError: You are trying to save a non contiguous tensor: encoder.block.0.layer.0.SelfAttention.q.weight which is not allowed. It either means you are trying to save tensors which are reference of each other in which case it's recommended to save only the full tensors, and reslice at load time, or simply call .contiguous() on your tensor to pack it before saving." }

  2. Assistant

    Эта ошибка происходит, когда вы пытаетесь сохранить тензор, который не является смежным. Несмежные тензоры имеют данные, которые не располагаются последовательно в памяти, что препятствует некоторым операциям, например, сохранению.

    Есть два способа решить эту проблему:

    1. Приведение к смежному виду: Прежде чем сохранять тензоры, вы можете вызвать метод .contiguous() на вашем тензоре. Это создаст копию тензора, в которой данные будут представлены последовательно в памяти. Пример:

      tensor = tensor.contiguous()
      
    2. Сохранение только полных тензоров и нарезка после загрузки: Если проблема связана с тем, что вы сохраняете ссылки на один и тот же тензор, попробуйте сохранять только полный тензор. После загрузки вы сможете нарезать его заново.

    Эти методы помогут избежать ошибок при сохранении тензоров, которые не являются смежными. Если вы используете библиотеку, которая управляет обучением и сохранением, как в случае с transformers, возможно, придется модифицировать сам фреймворк или следовать его рекомендациям для изменения процедуры сохранения.