Skip to content
TheChatArchiveThe Chat Archive
gpt-3.5-turbo-0125· OpenAIAi2 WildChat-4.8M (AI2)

bsz, qlen, = hiddenstates.size() querystates = self.qproj(hiddenstates) keystates =…

  1. User

    bsz, q_len, _ = hidden_states.size()

        query_states = self.q_proj(hidden_states)
        key_states = self.k_proj(hidden_states)
        value_states = self.v_proj(hidden_states)
    
        query_states = query_states.view(bsz, q_len, self.num_heads, self.head_dim).transpose(1, 2)
        key_states = key_states.view(bsz, q_len, self.num_key_value_heads, self.head_dim).transpose(1, 2)
        value_states = value_states.view(bsz, q_len, self.num_key_value_heads, self.head_dim).transpose(1, 2)
    
        past_key_value = getattr(self, "past_key_value", past_key_value)
        cos, sin = self.rotary_emb(value_states, position_ids, seq_len=None)
        query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin, None)
    
        if past_key_value is not None:
            # sin and cos are specific to RoPE models; position_ids needed for the static cache
            cache_kwargs = {"sin": sin, "cos": cos, "cache_position": cache_position}
            key_states, value_states = past_key_value.update(key_states, value_states, self.layer_idx, cache_kwargs)
    
        key_states = repeat_kv(key_states, self.num_key_value_groups)
        value_states = repeat_kv(value_states, self.num_key_value_groups)
    
        attn_weights = torch.matmul(query_states, key_states.transpose(2, 3)) / math.sqrt(self.head_dim)
    
        if attention_mask is not None:  # no matter the length, we just slice it
            if cache_position is not None:
                causal_mask = attention_mask[:, :, cache_position, : key_states.shape[-2]]
            else:
                causal_mask = attention_mask
            attn_weights = attn_weights + causal_mask
    
        # upcast attention to fp32
        attn_weights = nn.functional.softmax(attn_weights, dim=-1, dtype=torch.float32).to(query_states.dtype)
        attn_weights = nn.functional.dropout(attn_weights, p=self.attention_dropout, training=self.training)
        attn_output = torch.matmul(attn_weights, value_states)
    
        if attn_output.size() != (bsz, self.num_heads, q_len, self.head_dim):
            raise ValueError(
                f"`attn_output` should be of size {(bsz, self.num_heads, q_len, self.head_dim)}, but is"
                f" {attn_output.size()}"
            )
    
        attn_output = attn_output.transpose(1, 2).contiguous()
        attn_output = attn_output.reshape(bsz, q_len, self.hidden_size)
        # attn_output = attn_output.view(bsz, q_len, -1)
        attn_output = self.o_proj(attn_output)
    

    /usr/local/lib/python3.10/dist-packages/transformers/models/gemma/modeling_gemma.py in forward(self, hidden_states, attention_mask, position_ids, past_key_value, output_attentions, use_cache, cache_position, **kwargs) 276 277 attn_output = attn_output.transpose(1, 2).contiguous() --> 278 attn_output = attn_output.reshape(bsz, q_len, self.hidden_size) 279 # attn_output = attn_output.view(bsz, q_len, -1) 280 attn_output = self.o_proj(attn_output)

    RuntimeError: shape '[1, 6, 3072]' is invalid for input of size 24576

  2. Assistant

    The error is occurring because the reshaping of the attn_output tensor is not matching the expected shape (bsz, q_len, self.hidden_size). The tensor size of attn_output is 24576 while the expected size is [1, 6, 3072].

    To resolve this issue, you need to make sure that the reshaping operation is done correctly. Ensure that the dimensions of the input tensor attn_output are compatible with the target shape (bsz, q_len, self.hidden_size) before performing the reshape operation.

    Check the dimensions of the tensors involved in the reshaping operation and adjust the reshaping accordingly to match the expected shape (1, 6, 3072) for attn_output.

  3. User

    bsz, q_len, _ = hidden_states.size()

        query_states = self.q_proj(hidden_states)
        key_states = self.k_proj(hidden_states)
        value_states = self.v_proj(hidden_states)
    
        query_states = query_states.view(bsz, q_len, self.num_heads, self.head_dim).transpose(1, 2)
        key_states = key_states.view(bsz, q_len, self.num_key_value_heads, self.head_dim).transpose(1, 2)
        value_states = value_states.view(bsz, q_len, self.num_key_value_heads, self.head_dim).transpose(1, 2)
    
        past_key_value = getattr(self, "past_key_value", past_key_value)
        cos, sin = self.rotary_emb(value_states, position_ids, seq_len=None)
        query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin, None)
    
        if past_key_value is not None:
            # sin and cos are specific to RoPE models; position_ids needed for the static cache
            cache_kwargs = {"sin": sin, "cos": cos, "cache_position": cache_position}
            key_states, value_states = past_key_value.update(key_states, value_states, self.layer_idx, cache_kwargs)
    
        key_states = repeat_kv(key_states, self.num_key_value_groups)
        value_states = repeat_kv(value_states, self.num_key_value_groups)
    
        attn_weights = torch.matmul(query_states, key_states.transpose(2, 3)) / math.sqrt(self.head_dim)
    
        if attention_mask is not None:  # no matter the length, we just slice it
            if cache_position is not None:
                causal_mask = attention_mask[:, :, cache_position, : key_states.shape[-2]]
            else:
                causal_mask = attention_mask
            attn_weights = attn_weights + causal_mask
    
        # upcast attention to fp32
        attn_weights = nn.functional.softmax(attn_weights, dim=-1, dtype=torch.float32).to(query_states.dtype)
        attn_weights = nn.functional.dropout(attn_weights, p=self.attention_dropout, training=self.training)
        attn_output = torch.matmul(attn_weights, value_states)
    
        if attn_output.size() != (bsz, self.num_heads, q_len, self.head_dim):
            raise ValueError(
                f"`attn_output` should be of size {(bsz, self.num_heads, q_len, self.head_dim)}, but is"
                f" {attn_output.size()}"
            )
    
        attn_output = attn_output.transpose(1, 2).contiguous()
        attn_output = attn_output.reshape(bsz, q_len, self.hidden_size)
        # attn_output = attn_output.view(bsz, q_len, -1)
        attn_output = self.o_proj(attn_output)
    

    /usr/local/lib/python3.10/dist-packages/transformers/models/gemma/modeling_gemma.py in forward(self, hidden_states, attention_mask, position_ids, past_key_value, output_attentions, use_cache, cache_position, **kwargs) 276 277 attn_output = attn_output.transpose(1, 2).contiguous() --> 278 attn_output = attn_output.reshape(bsz, q_len, self.hidden_size) 279 # attn_output = attn_output.view(bsz, q_len, -1) 280 attn_output = self.o_proj(attn_output)

    RuntimeError: shape '[1, 6, 3072]' is invalid for input of size 24576 вычисли и распиши все shape над ошибкой, мне так легче будет

  4. Assistant

    Давайте вычислим и распишем все формы (shape), связанные с ошибкой, чтобы лучше понять, где возникла проблема. Давайте это сделаем.

    1. bsz, q_len, _ = hidden_states.size()
    2. query_states = query_states.view(bsz, q_len, self.num_heads, self.head_dim).transpose(1, 2)
      • Форма query_states: (bsz, num_heads, q_len, head_dim)
    3. key_states = key_states.view(bsz, q_len, self.num_key_value_heads, self.head_dim).transpose(1, 2)
      • Форма key_states: (bsz, num_key_value_heads, q_len, head_dim)
    4. value_states = value_states.view(bsz, q_len, self.num_key_value_heads, self.head_dim).transpose(1, 2)
      • Форма value_states: (bsz, num_key_value_heads, q_len, head_dim)
    5. attn_output = attn_output.reshape(bsz, q_len, self.hidden_size)
      • Форма attn_output: (bsz, q_len, hidden_size) ошибка в этой части кода
    6. if attn_output.size() != (bsz, self.num_heads, q_len, self.head_dim)
      • Форма attn_output: (bsz, num_heads, q_len, head_dim)
    7. o_proj принимает attn_output с формой (bsz, q_len, hidden_size)

    Ошибка происходит на шаге 5, где пытаемся изменить форму attn_output на (bsz, q_len, hidden_size). Видно, что ожидаемая форма (bsz, q_len, hidden_size) не соответствует фактической форме (bsz, num_heads, q_len, head_dim) на этом этапе.

    Похоже, что нужно пересмотреть преобразования форм при работе с query_states, key_states, value_states, и attn_output, чтобы они соответствовали ожидаемым формам для успешной работы модели.