Skip to content
The Chat Archive
gpt-4o-mini-2024-07-18· OpenAI WildChat-4.8M (AI2)

سلام. لطفا modelcard رو طبق فایل کانفیگ من بنویس، در مورد معماریش هم توضیح بده و برای…

  1. User

    سلام. لطفا model_card رو طبق فایل کانفیگ من بنویس، در مورد معماریش هم توضیح بده و برای هربخش مثالی هم بزن.{ "_name_or_path": "arshiaafshani/Arsh-V1", "architectures": [ "ArshForCausalLM" ], "auto_map": { "AutoModelForCausalLM": "modeling_arsh.ArshForCausalLM", "AutoConfig": "configuration_arsh.ArshConfig" }, "attention_bias": false, "attention_dropout": 0.0, "bos_token_id": 100257, "eos_token_id": 100265, "head_dim": 128, "hidden_act": "silu", "hidden_size": 5120, "initializer_range": 0.02, "intermediate_size": 17920, "max_position_embeddings": 16384, "mlp_bias": false, "model_type": "llama", "num_attention_heads": 40, "num_hidden_layers": 40, "num_key_value_heads": 10, "original_max_position_embeddings": 16384, "pad_token_id": 100351, "pretraining_tp": 1, "rms_norm_eps": 1e-05, "rope_scaling": null, "rope_theta": 250000, "tie_word_embeddings": false, "torch_dtype": "float16", "transformers_version": "4.47.1", "unsloth_fixed": true, "unsloth_version": "2025.1.8", "use_cache": true, "vocab_size": 100352 }

    فایل مدل: # coding=utf-8

    Copyright 2025 Arsh Team

    Apache License Version 2.0

    import torch import torch.nn as nn from transformers import ( PreTrainedModel, GenerationMixin, Cache, DynamicCache, StaticCache, FlashAttentionKwargs, ArshConfig ) from typing import Optional, Tuple, Union, List

    class ArshRMSNorm(nn.Module): """ RMS Normalization layer customized for Arsh architecture Args: hidden_size (int): Dimension of hidden states eps (float): Epsilon value for numerical stability Example: >>> norm = ArshRMSNorm(768) >>> x = torch.randn(1, 10, 768) >>> output = norm(x) """

    def __init__(self, hidden_size: int, eps: float = 1e-6):
        super().__init__()
        self.weight = nn.Parameter(torch.ones(hidden_size))
        self.variance_epsilon = eps
    
    def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
        input_dtype = hidden_states.dtype
        hidden_states = hidden_states.to(torch.float32)
        variance = hidden_states.pow(2).mean(-1, keepdim=True)
        hidden_states = hidden_states * torch.rsqrt(variance + self.variance_epsilon)
        return self.weight * hidden_states.to(input_dtype)
    

    class ArshRotaryEmbedding(nn.Module): """ Rotary Position Embedding implementation for Arsh model Args: config (ArshConfig): Model configuration Attributes: max_seq_len_cached (int): Maximum cached sequence length attention_scaling (float): Scaling factor for attention Example: >>> config = ArshConfig() >>> rotary_emb = ArshRotaryEmbedding(config) """

    def __init__(self, config: ArshConfig):
        super().__init__()
        self.config = config
        self.max_seq_len_cached = config.max_position_embeddings
        self.rope_type = config.rope_scaling.get("type", "default")
        self.rope_init_fn = ROPE_INIT_FUNCTIONS[self.rope_type]
    
        inv_freq, self.attention_scaling = self.rope_init_fn(config)
        self.register_buffer("inv_freq", inv_freq, persistent=False)
    
    def _update_frequency(self, position_ids: torch.Tensor):
        """Dynamically update frequency based on input sequence length"""
        seq_len = position_ids.max() + 1
        if seq_len > self.max_seq_len_cached:
            inv_freq, self.attention_scaling = self.rope_init_fn(
                self.config, seq_len=seq_len
            )
            self.register_buffer("inv_freq", inv_freq)
            self.max_seq_len_cached = seq_len
    
    def forward(self, x: torch.Tensor, position_ids: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]:
        if "dynamic" in self.rope_type:
            self._update_frequency(position_ids)
    
        # Compute cosine and sine embeddings
        inv_freq_expanded = self.inv_freq[None, :, None]
        position_ids_expanded = position_ids[:, None, :].float()
    
        with torch.autocast(device_type=x.device.type, enabled=False):
            freqs = (inv_freq_expanded @ position_ids_expanded).transpose(1, 2)
            emb = torch.cat((freqs, freqs), dim=-1)
            cos = emb.cos()
            sin = emb.sin()
    
        return cos * self.attention_scaling, sin * self.attention_scaling
    

    class ArshMLP(nn.Module): """ Gated MLP Block for Arsh model Args: config (ArshConfig): Model configuration Example: >>> config = ArshConfig() >>> mlp = ArshMLP(config) >>> x = torch.randn(1, 10, 768) >>> output = mlp(x) """

    def __init__(self, config: ArshConfig):
        super().__init__()
        self.gate_proj = nn.Linear(config.hidden_size, config.intermediate_size)
        self.up_proj = nn.Linear(config.hidden_size, config.intermediate_size)
        self.down_proj = nn.Linear(config.intermediate_size, config.hidden_size)
        self.act_fn = ACT2FN[config.hidden_act]
    
    def forward(self, x: torch.Tensor) -> torch.Tensor:
        return self.down_proj(self.act_fn(self.gate_proj(x)) * self.up_proj(x))
    

    class ArshAttention(nn.Module): """ Multi-head Attention layer with RoPE support Args: config (ArshConfig): Model configuration layer_idx (int): Layer index Example: >>> config = ArshConfig() >>> attn = ArshAttention(config, layer_idx=0) """

    def __init__(self, config: ArshConfig, layer_idx: int):
        super().__init__()
        self.config = config
        self.layer_idx = layer_idx
        self.hidden_size = config.hidden_size
        self.num_heads = config.num_attention_heads
        self.head_dim = self.hidden_size // self.num_heads
    
        self.q_proj = nn.Linear(self.hidden_size, self.num_heads * self.head_dim)
        self.k_proj = nn.Linear(self.hidden_size, self.num_heads * self.head_dim)
        self.v_proj = nn.Linear(self.hidden_size, self.num_heads * self.head_dim)
        self.o_proj = nn.Linear(self.num_heads * self.head_dim, self.hidden_size)
    
        self.rotary_emb = ArshRotaryEmbedding(config)
    
    def forward(
            self,
            hidden_states: torch.Tensor,
            attention_mask: Optional[torch.Tensor] = None,
            position_ids: Optional[torch.Tensor] = None,
    ) -> Tuple[torch.Tensor, torch.Tensor]:
        batch_size, seq_len, _ = hidden_states.shape
    
        # Project inputs
        q = self.q_proj(hidden_states).view(batch_size, seq_len, self.num_heads, self.head_dim)
        k = self.k_proj(hidden_states).view(batch_size, seq_len, self.num_heads, self.head_dim)
        v = self.v_proj(hidden_states).view(batch_size, seq_len, self.num_heads, self.head_dim)
    
        # Apply rotary embeddings
        cos, sin = self.rotary_emb(hidden_states, position_ids)
        q, k = apply_rotary_pos_emb(q, k, cos, sin)
    
        # Attention computation
        attn_output, attn_weights = scaled_dot_product_attention(
            q, k, v, attention_mask=attention_mask
        )
    
        # Output projection
        output = self.o_proj(attn_output.view(batch_size, seq_len, -1))
        return output, attn_weights
    

    class ArshDecoderLayer(nn.Module): """ Transformer Decoder Layer Args: config (ArshConfig): Model configuration layer_idx (int): Layer index Example: >>> config = ArshConfig() >>> layer = ArshDecoderLayer(config, layer_idx=0) """

    def __init__(self, config: ArshConfig, layer_idx: int):
        super().__init__()
        self.self_attn = ArshAttention(config, layer_idx)
        self.mlp = ArshMLP(config)
        self.input_norm = ArshRMSNorm(config.hidden_size)
        self.post_attn_norm = ArshRMSNorm(config.hidden_size)
    
    def forward(
            self,
            hidden_states: torch.Tensor,
            attention_mask: Optional[torch.Tensor] = None,
            position_ids: Optional[torch.Tensor] = None,
    ) -> torch.Tensor:
        # Self-attention block
        residual = hidden_states
        hidden_states = self.input_norm(hidden_states)
        attn_output, _ = self.self_attn(hidden_states, attention_mask, position_ids)
        hidden_states = residual + attn_output
    
        # MLP block
        residual = hidden_states
        hidden_states = self.post_attn_norm(hidden_states)
        hidden_states = self.mlp(hidden_states)
        hidden_states = residual + hidden_states
    
        return hidden_states
    

    class ArshModel(PreTrainedModel): """ Main Arsh model architecture Args: config (ArshConfig): Model configuration Example: >>> config = ArshConfig() >>> model = ArshModel(config) """

    def __init__(self, config: ArshConfig):
        super().__init__(config)
        self.embed_tokens = nn.Embedding(config.vocab_size, config.hidden_size)
        self.layers = nn.ModuleList(
            [ArshDecoderLayer(config, i) for i in range(config.num_hidden_layers)]
        )
        self.norm = ArshRMSNorm(config.hidden_size)
    
        self.post_init()
    
    def forward(
            self,
            input_ids: torch.Tensor,
            attention_mask: Optional[torch.Tensor] = None,
            position_ids: Optional[torch.Tensor] = None,
    ) -> torch.Tensor:
        hidden_states = self.embed_tokens(input_ids)
    
        for layer in self.layers:
            hidden_states = layer(
                hidden_states,
                attention_mask=attention_mask,
                position_ids=position_ids
            )
    
        return self.norm(hidden_states)
    

    class ArshForCausalLM(ArshModel, GenerationMixin): """ Arsh model for causal language modeling Args: config (ArshConfig): Model configuration Example: >>> config = ArshConfig() >>> model = ArshForCausalLM(config) >>> inputs = {"input_ids": torch.randint(0, 100, (1, 10))} >>> outputs = model(**inputs) """

    def __init__(self, config: ArshConfig):
        super().__init__(config)
        self.lm_head = nn.Linear(config.hidden_size, config.vocab_size, bias=False)
        self.post_init()
    
    def forward(
            self,
            input_ids: torch.Tensor,
            attention_mask: Optional[torch.Tensor] = None,
            labels: Optional[torch.Tensor] = None,
    ) -> dict:
        hidden_states = super().forward(input_ids, attention_mask)
        logits = self.lm_head(hidden_states)
    
        loss = None
        if labels is not None:
            loss_fct = nn.CrossEntropyLoss()
            loss = loss_fct(logits.view(-1, self.config.vocab_size), labels.view(-1))
    
        return {"loss": loss, "logits": logits}
    

    model_card من ( readme.md): --- license: mit pipeline_tag: text-generation library_name: transformers tags:

    • safetensors

    We introduce our first reasoning and chat model based on phi weights. We use Arsh architecture based on Llama to make our model usable on Unsloth. We used some clear and good data to train this model and we made lots of things to make this model. We are going to have lots of updates every day and some mixtures of architectures to make the best model ever. Now we are so close to do that but we need more time to work on our model

  2. Assistant
    ---
    license: mit
    pipeline_tag: text-generation
    library_name: transformers
    tags:
    - safetensors
    ---
    
    # Arsh-V1 Model Card
    
    We introduce our first reasoning and chat model based on phi weights. The Arsh architecture is built on the Llama framework, making this model optimized for performance and extendability in chat applications. Our team has utilized high-quality datasets for training to ensure reliable and robust performance.
    
    Our ongoing efforts include frequent updates and enhancements that aim to blend multiple architectures, striving to build the best model possible. We are making significant progress and are dedicated to refining Arsh-V1 further.
    
    ---
    
    ## Model Overview
    
    The Arsh-V1 model belongs to the `Causal Language Model (LLM)` family and is designed for text generation tasks. It comprises a stack of transformer blocks and is optimized for high throughput and low latency in language generation.
    
    ### Architecture
    
    - **Model Type:** Llama
    - **Hidden Size:** 5120
    - **Number of Layers:** 40
    - **Attention Heads:** 40
    - **Maximum Sequence Length:** 16384
    - **Vocabulary Size:** 100352
    - **Activation Function:** SiLU (Gaussian Error Linear Unit)
    
    This architecture enables the model to capture complex language patterns, making it suitable for conversation and reasoning tasks.
    
    ---
    
    ## Key Components
    
    ### 1. **RMS Normalization**
    
    The `ArshRMSNorm` layer is employed for normalization within the model, enhancing training stability and speed.
    
    ```python
    norm = ArshRMSNorm(hidden_size=5120)
    x = torch.randn(1, 10, 5120)
    output = norm(x)
    

    2. Rotary Position Embedding

    ArshRotaryEmbedding leverages rotary embeddings for efficient positional encoding in transformer architectures.

    config = ArshConfig(max_position_embeddings=16384)
    rotary_emb = ArshRotaryEmbedding(config)
    

    3. Gated MLP Block

    The ArshMLP component is responsible for non-linear transformations, incorporating a gating mechanism.

    mlp = ArshMLP(config)
    x = torch.randn(1, 10, 5120)
    output = mlp(x)
    

    4. Multi-Head Attention

    The ArshAttention layer implements multi-head attention with support for rotary positional embeddings, enhancing context understanding.

    attention_layer = ArshAttention(config, layer_idx=0)
    hidden_states = torch.randn(1, 10, 5120)
    attn_output, attn_weights = attention_layer(hidden_states)
    

    5. Transformer Decoder Layer

    The ArshDecoderLayer integrates self-attention and feed-forward neural network components in series.

    decoder_layer = ArshDecoderLayer(config, layer_idx=0)
    hidden_states = decoder_layer(hidden_states)
    

    Usage

    Arsh-V1 can be utilized for a variety of text generation tasks, such as conversational agents, storytelling, and more.

    Example

    from transformers import ArshForCausalLM
    
    # Initialize the model
    model = ArshForCausalLM(config)
    
    # Prepare input
    input_ids = torch.randint(0, 100352, (1, 10))
    
    # Generate outputs
    outputs = model(input_ids)
    

    License

    Our model is based on MIT license, which allows for modifications, creation, fine-tuning, and commercial usage. We appreciate your contributions to make Arsh-V1 better!