bsz, qlen, = hiddenstates.size() querystates = self.qproj(hiddenstates) keystates =…
- User
bsz, q_len, _ = hidden_states.size()
query_states = self.q_proj(hidden_states) key_states = self.k_proj(hidden_states) value_states = self.v_proj(hidden_states) query_states = query_states.view(bsz, q_len, self.num_heads, self.head_dim).transpose(1, 2) key_states = key_states.view(bsz, q_len, self.num_key_value_heads, self.head_dim).transpose(1, 2) value_states = value_states.view(bsz, q_len, self.num_key_value_heads, self.head_dim).transpose(1, 2) past_key_value = getattr(self, "past_key_value", past_key_value) cos, sin = self.rotary_emb(value_states, position_ids, seq_len=None) query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin, None) if past_key_value is not None: # sin and cos are specific to RoPE models; position_ids needed for the static cache cache_kwargs = {"sin": sin, "cos": cos, "cache_position": cache_position} key_states, value_states = past_key_value.update(key_states, value_states, self.layer_idx, cache_kwargs) key_states = repeat_kv(key_states, self.num_key_value_groups) value_states = repeat_kv(value_states, self.num_key_value_groups) attn_weights = torch.matmul(query_states, key_states.transpose(2, 3)) / math.sqrt(self.head_dim) if attention_mask is not None: # no matter the length, we just slice it if cache_position is not None: causal_mask = attention_mask[:, :, cache_position, : key_states.shape[-2]] else: causal_mask = attention_mask attn_weights = attn_weights + causal_mask # upcast attention to fp32 attn_weights = nn.functional.softmax(attn_weights, dim=-1, dtype=torch.float32).to(query_states.dtype) attn_weights = nn.functional.dropout(attn_weights, p=self.attention_dropout, training=self.training) attn_output = torch.matmul(attn_weights, value_states) if attn_output.size() != (bsz, self.num_heads, q_len, self.head_dim): raise ValueError( f"`attn_output` should be of size {(bsz, self.num_heads, q_len, self.head_dim)}, but is" f" {attn_output.size()}" ) attn_output = attn_output.transpose(1, 2).contiguous() attn_output = attn_output.reshape(bsz, q_len, self.hidden_size) # attn_output = attn_output.view(bsz, q_len, -1) attn_output = self.o_proj(attn_output)/usr/local/lib/python3.10/dist-packages/transformers/models/gemma/modeling_gemma.py in forward(self, hidden_states, attention_mask, position_ids, past_key_value, output_attentions, use_cache, cache_position, **kwargs) 276 277 attn_output = attn_output.transpose(1, 2).contiguous() --> 278 attn_output = attn_output.reshape(bsz, q_len, self.hidden_size) 279 # attn_output = attn_output.view(bsz, q_len, -1) 280 attn_output = self.o_proj(attn_output)
RuntimeError: shape '[1, 6, 3072]' is invalid for input of size 24576
- Assistant
The error is occurring because the reshaping of the
attn_outputtensor is not matching the expected shape(bsz, q_len, self.hidden_size). The tensor size ofattn_outputis 24576 while the expected size is[1, 6, 3072].To resolve this issue, you need to make sure that the reshaping operation is done correctly. Ensure that the dimensions of the input tensor
attn_outputare compatible with the target shape(bsz, q_len, self.hidden_size)before performing the reshape operation.Check the dimensions of the tensors involved in the reshaping operation and adjust the reshaping accordingly to match the expected shape
(1, 6, 3072)forattn_output. - User
bsz, q_len, _ = hidden_states.size()
query_states = self.q_proj(hidden_states) key_states = self.k_proj(hidden_states) value_states = self.v_proj(hidden_states) query_states = query_states.view(bsz, q_len, self.num_heads, self.head_dim).transpose(1, 2) key_states = key_states.view(bsz, q_len, self.num_key_value_heads, self.head_dim).transpose(1, 2) value_states = value_states.view(bsz, q_len, self.num_key_value_heads, self.head_dim).transpose(1, 2) past_key_value = getattr(self, "past_key_value", past_key_value) cos, sin = self.rotary_emb(value_states, position_ids, seq_len=None) query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin, None) if past_key_value is not None: # sin and cos are specific to RoPE models; position_ids needed for the static cache cache_kwargs = {"sin": sin, "cos": cos, "cache_position": cache_position} key_states, value_states = past_key_value.update(key_states, value_states, self.layer_idx, cache_kwargs) key_states = repeat_kv(key_states, self.num_key_value_groups) value_states = repeat_kv(value_states, self.num_key_value_groups) attn_weights = torch.matmul(query_states, key_states.transpose(2, 3)) / math.sqrt(self.head_dim) if attention_mask is not None: # no matter the length, we just slice it if cache_position is not None: causal_mask = attention_mask[:, :, cache_position, : key_states.shape[-2]] else: causal_mask = attention_mask attn_weights = attn_weights + causal_mask # upcast attention to fp32 attn_weights = nn.functional.softmax(attn_weights, dim=-1, dtype=torch.float32).to(query_states.dtype) attn_weights = nn.functional.dropout(attn_weights, p=self.attention_dropout, training=self.training) attn_output = torch.matmul(attn_weights, value_states) if attn_output.size() != (bsz, self.num_heads, q_len, self.head_dim): raise ValueError( f"`attn_output` should be of size {(bsz, self.num_heads, q_len, self.head_dim)}, but is" f" {attn_output.size()}" ) attn_output = attn_output.transpose(1, 2).contiguous() attn_output = attn_output.reshape(bsz, q_len, self.hidden_size) # attn_output = attn_output.view(bsz, q_len, -1) attn_output = self.o_proj(attn_output)/usr/local/lib/python3.10/dist-packages/transformers/models/gemma/modeling_gemma.py in forward(self, hidden_states, attention_mask, position_ids, past_key_value, output_attentions, use_cache, cache_position, **kwargs) 276 277 attn_output = attn_output.transpose(1, 2).contiguous() --> 278 attn_output = attn_output.reshape(bsz, q_len, self.hidden_size) 279 # attn_output = attn_output.view(bsz, q_len, -1) 280 attn_output = self.o_proj(attn_output)
RuntimeError: shape '[1, 6, 3072]' is invalid for input of size 24576 вычисли и распиши все shape над ошибкой, мне так легче будет
- Assistant
Давайте вычислим и распишем все формы (shape), связанные с ошибкой, чтобы лучше понять, где возникла проблема. Давайте это сделаем.
bsz, q_len, _ = hidden_states.size()query_states = query_states.view(bsz, q_len, self.num_heads, self.head_dim).transpose(1, 2)- Форма
query_states:(bsz, num_heads, q_len, head_dim)
- Форма
key_states = key_states.view(bsz, q_len, self.num_key_value_heads, self.head_dim).transpose(1, 2)- Форма
key_states:(bsz, num_key_value_heads, q_len, head_dim)
- Форма
value_states = value_states.view(bsz, q_len, self.num_key_value_heads, self.head_dim).transpose(1, 2)- Форма
value_states:(bsz, num_key_value_heads, q_len, head_dim)
- Форма
attn_output = attn_output.reshape(bsz, q_len, self.hidden_size)- Форма
attn_output:(bsz, q_len, hidden_size)ошибка в этой части кода
- Форма
if attn_output.size() != (bsz, self.num_heads, q_len, self.head_dim)- Форма
attn_output:(bsz, num_heads, q_len, head_dim)
- Форма
o_projпринимаетattn_outputс формой(bsz, q_len, hidden_size)
Ошибка происходит на шаге 5, где пытаемся изменить форму
attn_outputна(bsz, q_len, hidden_size). Видно, что ожидаемая форма(bsz, q_len, hidden_size)не соответствует фактической форме(bsz, num_heads, q_len, head_dim)на этом этапе.Похоже, что нужно пересмотреть преобразования форм при работе с
query_states,key_states,value_states, иattn_output, чтобы они соответствовали ожидаемым формам для успешной работы модели.