سلام. لطفا modelcard رو طبق فایل کانفیگ من بنویس، در مورد معماریش هم توضیح بده و برای…
- User
سلام. لطفا model_card رو طبق فایل کانفیگ من بنویس، در مورد معماریش هم توضیح بده و برای هربخش مثالی هم بزن.{ "_name_or_path": "arshiaafshani/Arsh-V1", "architectures": [ "ArshForCausalLM" ], "auto_map": { "AutoModelForCausalLM": "modeling_arsh.ArshForCausalLM", "AutoConfig": "configuration_arsh.ArshConfig" }, "attention_bias": false, "attention_dropout": 0.0, "bos_token_id": 100257, "eos_token_id": 100265, "head_dim": 128, "hidden_act": "silu", "hidden_size": 5120, "initializer_range": 0.02, "intermediate_size": 17920, "max_position_embeddings": 16384, "mlp_bias": false, "model_type": "llama", "num_attention_heads": 40, "num_hidden_layers": 40, "num_key_value_heads": 10, "original_max_position_embeddings": 16384, "pad_token_id": 100351, "pretraining_tp": 1, "rms_norm_eps": 1e-05, "rope_scaling": null, "rope_theta": 250000, "tie_word_embeddings": false, "torch_dtype": "float16", "transformers_version": "4.47.1", "unsloth_fixed": true, "unsloth_version": "2025.1.8", "use_cache": true, "vocab_size": 100352 }
فایل مدل: # coding=utf-8
Copyright 2025 Arsh Team
Apache License Version 2.0
import torch import torch.nn as nn from transformers import ( PreTrainedModel, GenerationMixin, Cache, DynamicCache, StaticCache, FlashAttentionKwargs, ArshConfig ) from typing import Optional, Tuple, Union, List
class ArshRMSNorm(nn.Module): """ RMS Normalization layer customized for Arsh architecture Args: hidden_size (int): Dimension of hidden states eps (float): Epsilon value for numerical stability Example: >>> norm = ArshRMSNorm(768) >>> x = torch.randn(1, 10, 768) >>> output = norm(x) """
def __init__(self, hidden_size: int, eps: float = 1e-6): super().__init__() self.weight = nn.Parameter(torch.ones(hidden_size)) self.variance_epsilon = eps def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: input_dtype = hidden_states.dtype hidden_states = hidden_states.to(torch.float32) variance = hidden_states.pow(2).mean(-1, keepdim=True) hidden_states = hidden_states * torch.rsqrt(variance + self.variance_epsilon) return self.weight * hidden_states.to(input_dtype)class ArshRotaryEmbedding(nn.Module): """ Rotary Position Embedding implementation for Arsh model Args: config (ArshConfig): Model configuration Attributes: max_seq_len_cached (int): Maximum cached sequence length attention_scaling (float): Scaling factor for attention Example: >>> config = ArshConfig() >>> rotary_emb = ArshRotaryEmbedding(config) """
def __init__(self, config: ArshConfig): super().__init__() self.config = config self.max_seq_len_cached = config.max_position_embeddings self.rope_type = config.rope_scaling.get("type", "default") self.rope_init_fn = ROPE_INIT_FUNCTIONS[self.rope_type] inv_freq, self.attention_scaling = self.rope_init_fn(config) self.register_buffer("inv_freq", inv_freq, persistent=False) def _update_frequency(self, position_ids: torch.Tensor): """Dynamically update frequency based on input sequence length""" seq_len = position_ids.max() + 1 if seq_len > self.max_seq_len_cached: inv_freq, self.attention_scaling = self.rope_init_fn( self.config, seq_len=seq_len ) self.register_buffer("inv_freq", inv_freq) self.max_seq_len_cached = seq_len def forward(self, x: torch.Tensor, position_ids: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]: if "dynamic" in self.rope_type: self._update_frequency(position_ids) # Compute cosine and sine embeddings inv_freq_expanded = self.inv_freq[None, :, None] position_ids_expanded = position_ids[:, None, :].float() with torch.autocast(device_type=x.device.type, enabled=False): freqs = (inv_freq_expanded @ position_ids_expanded).transpose(1, 2) emb = torch.cat((freqs, freqs), dim=-1) cos = emb.cos() sin = emb.sin() return cos * self.attention_scaling, sin * self.attention_scalingclass ArshMLP(nn.Module): """ Gated MLP Block for Arsh model Args: config (ArshConfig): Model configuration Example: >>> config = ArshConfig() >>> mlp = ArshMLP(config) >>> x = torch.randn(1, 10, 768) >>> output = mlp(x) """
def __init__(self, config: ArshConfig): super().__init__() self.gate_proj = nn.Linear(config.hidden_size, config.intermediate_size) self.up_proj = nn.Linear(config.hidden_size, config.intermediate_size) self.down_proj = nn.Linear(config.intermediate_size, config.hidden_size) self.act_fn = ACT2FN[config.hidden_act] def forward(self, x: torch.Tensor) -> torch.Tensor: return self.down_proj(self.act_fn(self.gate_proj(x)) * self.up_proj(x))class ArshAttention(nn.Module): """ Multi-head Attention layer with RoPE support Args: config (ArshConfig): Model configuration layer_idx (int): Layer index Example: >>> config = ArshConfig() >>> attn = ArshAttention(config, layer_idx=0) """
def __init__(self, config: ArshConfig, layer_idx: int): super().__init__() self.config = config self.layer_idx = layer_idx self.hidden_size = config.hidden_size self.num_heads = config.num_attention_heads self.head_dim = self.hidden_size // self.num_heads self.q_proj = nn.Linear(self.hidden_size, self.num_heads * self.head_dim) self.k_proj = nn.Linear(self.hidden_size, self.num_heads * self.head_dim) self.v_proj = nn.Linear(self.hidden_size, self.num_heads * self.head_dim) self.o_proj = nn.Linear(self.num_heads * self.head_dim, self.hidden_size) self.rotary_emb = ArshRotaryEmbedding(config) def forward( self, hidden_states: torch.Tensor, attention_mask: Optional[torch.Tensor] = None, position_ids: Optional[torch.Tensor] = None, ) -> Tuple[torch.Tensor, torch.Tensor]: batch_size, seq_len, _ = hidden_states.shape # Project inputs q = self.q_proj(hidden_states).view(batch_size, seq_len, self.num_heads, self.head_dim) k = self.k_proj(hidden_states).view(batch_size, seq_len, self.num_heads, self.head_dim) v = self.v_proj(hidden_states).view(batch_size, seq_len, self.num_heads, self.head_dim) # Apply rotary embeddings cos, sin = self.rotary_emb(hidden_states, position_ids) q, k = apply_rotary_pos_emb(q, k, cos, sin) # Attention computation attn_output, attn_weights = scaled_dot_product_attention( q, k, v, attention_mask=attention_mask ) # Output projection output = self.o_proj(attn_output.view(batch_size, seq_len, -1)) return output, attn_weightsclass ArshDecoderLayer(nn.Module): """ Transformer Decoder Layer Args: config (ArshConfig): Model configuration layer_idx (int): Layer index Example: >>> config = ArshConfig() >>> layer = ArshDecoderLayer(config, layer_idx=0) """
def __init__(self, config: ArshConfig, layer_idx: int): super().__init__() self.self_attn = ArshAttention(config, layer_idx) self.mlp = ArshMLP(config) self.input_norm = ArshRMSNorm(config.hidden_size) self.post_attn_norm = ArshRMSNorm(config.hidden_size) def forward( self, hidden_states: torch.Tensor, attention_mask: Optional[torch.Tensor] = None, position_ids: Optional[torch.Tensor] = None, ) -> torch.Tensor: # Self-attention block residual = hidden_states hidden_states = self.input_norm(hidden_states) attn_output, _ = self.self_attn(hidden_states, attention_mask, position_ids) hidden_states = residual + attn_output # MLP block residual = hidden_states hidden_states = self.post_attn_norm(hidden_states) hidden_states = self.mlp(hidden_states) hidden_states = residual + hidden_states return hidden_statesclass ArshModel(PreTrainedModel): """ Main Arsh model architecture Args: config (ArshConfig): Model configuration Example: >>> config = ArshConfig() >>> model = ArshModel(config) """
def __init__(self, config: ArshConfig): super().__init__(config) self.embed_tokens = nn.Embedding(config.vocab_size, config.hidden_size) self.layers = nn.ModuleList( [ArshDecoderLayer(config, i) for i in range(config.num_hidden_layers)] ) self.norm = ArshRMSNorm(config.hidden_size) self.post_init() def forward( self, input_ids: torch.Tensor, attention_mask: Optional[torch.Tensor] = None, position_ids: Optional[torch.Tensor] = None, ) -> torch.Tensor: hidden_states = self.embed_tokens(input_ids) for layer in self.layers: hidden_states = layer( hidden_states, attention_mask=attention_mask, position_ids=position_ids ) return self.norm(hidden_states)class ArshForCausalLM(ArshModel, GenerationMixin): """ Arsh model for causal language modeling Args: config (ArshConfig): Model configuration Example: >>> config = ArshConfig() >>> model = ArshForCausalLM(config) >>> inputs = {"input_ids": torch.randint(0, 100, (1, 10))} >>> outputs = model(**inputs) """
def __init__(self, config: ArshConfig): super().__init__(config) self.lm_head = nn.Linear(config.hidden_size, config.vocab_size, bias=False) self.post_init() def forward( self, input_ids: torch.Tensor, attention_mask: Optional[torch.Tensor] = None, labels: Optional[torch.Tensor] = None, ) -> dict: hidden_states = super().forward(input_ids, attention_mask) logits = self.lm_head(hidden_states) loss = None if labels is not None: loss_fct = nn.CrossEntropyLoss() loss = loss_fct(logits.view(-1, self.config.vocab_size), labels.view(-1)) return {"loss": loss, "logits": logits}model_card من ( readme.md): --- license: mit pipeline_tag: text-generation library_name: transformers tags:
- safetensors
We introduce our first reasoning and chat model based on phi weights. We use Arsh architecture based on Llama to make our model usable on Unsloth. We used some clear and good data to train this model and we made lots of things to make this model. We are going to have lots of updates every day and some mixtures of architectures to make the best model ever. Now we are so close to do that but we need more time to work on our model
- Assistant
--- license: mit pipeline_tag: text-generation library_name: transformers tags: - safetensors --- # Arsh-V1 Model Card We introduce our first reasoning and chat model based on phi weights. The Arsh architecture is built on the Llama framework, making this model optimized for performance and extendability in chat applications. Our team has utilized high-quality datasets for training to ensure reliable and robust performance. Our ongoing efforts include frequent updates and enhancements that aim to blend multiple architectures, striving to build the best model possible. We are making significant progress and are dedicated to refining Arsh-V1 further. --- ## Model Overview The Arsh-V1 model belongs to the `Causal Language Model (LLM)` family and is designed for text generation tasks. It comprises a stack of transformer blocks and is optimized for high throughput and low latency in language generation. ### Architecture - **Model Type:** Llama - **Hidden Size:** 5120 - **Number of Layers:** 40 - **Attention Heads:** 40 - **Maximum Sequence Length:** 16384 - **Vocabulary Size:** 100352 - **Activation Function:** SiLU (Gaussian Error Linear Unit) This architecture enables the model to capture complex language patterns, making it suitable for conversation and reasoning tasks. --- ## Key Components ### 1. **RMS Normalization** The `ArshRMSNorm` layer is employed for normalization within the model, enhancing training stability and speed. ```python norm = ArshRMSNorm(hidden_size=5120) x = torch.randn(1, 10, 5120) output = norm(x)2. Rotary Position Embedding
ArshRotaryEmbeddingleverages rotary embeddings for efficient positional encoding in transformer architectures.config = ArshConfig(max_position_embeddings=16384) rotary_emb = ArshRotaryEmbedding(config)3. Gated MLP Block
The
ArshMLPcomponent is responsible for non-linear transformations, incorporating a gating mechanism.mlp = ArshMLP(config) x = torch.randn(1, 10, 5120) output = mlp(x)4. Multi-Head Attention
The
ArshAttentionlayer implements multi-head attention with support for rotary positional embeddings, enhancing context understanding.attention_layer = ArshAttention(config, layer_idx=0) hidden_states = torch.randn(1, 10, 5120) attn_output, attn_weights = attention_layer(hidden_states)5. Transformer Decoder Layer
The
ArshDecoderLayerintegrates self-attention and feed-forward neural network components in series.decoder_layer = ArshDecoderLayer(config, layer_idx=0) hidden_states = decoder_layer(hidden_states)
Usage
Arsh-V1 can be utilized for a variety of text generation tasks, such as conversational agents, storytelling, and more.
Example
from transformers import ArshForCausalLM # Initialize the model model = ArshForCausalLM(config) # Prepare input input_ids = torch.randint(0, 100352, (1, 10)) # Generate outputs outputs = model(input_ids)
License
Our model is based on MIT license, which allows for modifications, creation, fine-tuning, and commercial usage. We appreciate your contributions to make Arsh-V1 better!