Skip to content
The Chat Archive
gemini-2.0-flash-001· Google LMArena Human Preference 140K

https://arxiv.org/pdf/2409.02060 import time import numpy as np…

  1. User

    https://arxiv.org/pdf/2409.02060

    import time import numpy as np np.set_printoptions(suppress=True, linewidth=1000) import functools from tinygrad import Tensor, nn, Device, GlobalCounters from tinygrad.helpers import Timing, getenv from extra.models.llama import Transformer, convert_from_huggingface

    class MixtureFeedForward: def init(self, num_experts:int, activated_experts:int, dim:int, hidden_dim:int, linear=nn.Linear): self.activated_experts = activated_experts self.gate = nn.Linear(dim, num_experts, bias=False) self.up_proj = Tensor.zeros(num_experts, hidden_dim, dim, dtype='bfloat16') self.down_proj = Tensor.zeros(num_experts, dim, hidden_dim, dtype='bfloat16') self.gate_proj = Tensor.zeros(num_experts, hidden_dim, dim, dtype='bfloat16') def call(self, x:Tensor) -> Tensor: assert x.shape[0] == 1, "only BS=1" assert x.shape[1] == 1, "only length=1" g = self.gate(x).float().softmax(-1)

    g = g.squeeze() # (BS, length, num_experts) -> (num_experts,)
    probs, sel = g.topk(self.activated_experts)
    
    # run MoE
    x_up_gate = x.dot(self.gate_proj[sel].permute(0,2,1)).silu() * x.dot(self.up_proj[sel].permute(0,2,1))
    x_down = x_up_gate.dot(self.down_proj[sel].permute(0,2,1))
    return (x_down.float() * probs.reshape(self.activated_experts, 1, 1)).sum(axis=0)
    

    model is bf16, 1.3B active, 6.9B total

    M3 Max is 400 GB/s, so 400/2.6 = ~154 tok/s

    def fetch_weights() -> dict[str, Tensor]:

    TODO: make this lazy so the 3 fetches can happen in parallel

    m1 = Tensor.from_url("https://huggingface.co/allenai/OLMoE-1B-7B-0924/resolve/main/model-00001-of-00003.safetensors").to(Device.DEFAULT) m2 = Tensor.from_url("https://huggingface.co/allenai/OLMoE-1B-7B-0924/resolve/main/model-00002-of-00003.safetensors").to(Device.DEFAULT) m3 = Tensor.from_url("https://huggingface.co/allenai/OLMoE-1B-7B-0924/resolve/main/model-00003-of-00003.safetensors").to(Device.DEFAULT) return {**nn.state.safe_load(m1), **nn.state.safe_load(m2), **nn.state.safe_load(m3)}

    if name == "main": if getenv("TORCH"): from transformers import OlmoeForCausalLM, AutoTokenizer model = OlmoeForCausalLM.from_pretrained("allenai/OLMoE-1B-7B-0924") tokenizer = AutoTokenizer.from_pretrained("allenai/OLMoE-1B-7B-0924") inputs = tokenizer("Hello", return_tensors="pt") generate_ids = model.generate(inputs.input_ids, max_length=30) out = tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0] print(out) exit(0)

    with Timing("create model: "): model = Transformer(n_layers=16, dim=2048, hidden_dim=1024, n_heads=16, norm_eps=1e-5, qk_norm=1e-5, max_context=1024, vocab_size=50304, feed_forward=functools.partial(MixtureFeedForward, 64, 8)) model_state_dict = nn.state.get_state_dict(model) del model_state_dict['freqs_cis']

    with Timing("load weights to GPU: "): nhf_state = convert_from_huggingface(fetch_weights(), model, 16, 16) # NOTE: i'm not sure this actually needs float32, it may just change the type of things downstream from it. but doesn't match torch w/o this for needs_float32 in ['tok_embeddings.weight']: nhf_state[needs_float32] = nhf_state[needs_float32].float() print(f"ram used: {GlobalCounters.mem_used/1e9:.2f} GB")

    with Timing("unpack weights: "): nn.state.load_state_dict(model, nhf_state, verbose=False, strict=False, consume=True, realize=False) assert len(nhf_state) == 0 Tensor.realize(*list(nn.state.get_state_dict(model).values())) print(f"ram used: {GlobalCounters.mem_used/1e9:.2f} GB")

    count = 30 temperature = 0

    with Timing("load tokenizer: "): from transformers import AutoTokenizer tokenizer = AutoTokenizer.from_pretrained("allenai/OLMoE-1B-7B-0924")

    toks = [12092] start_pos = 0 timings = [] for i in range(count): GlobalCounters.reset() st = time.perf_counter() tok = model(Tensor([toks[start_pos:]]), start_pos, temperature).item() timings.append(time.perf_counter()-st) toks.append(tok) start_pos += 1 print(toks) print(tokenizer.decode(toks)) print(f"fastest token {min(timings)*1e3:.2f} ms, {1/min(timings):.1f} tok/s")

    if temperature == 0: # Hello, I am a newbie to this forum and I am trying to get a better understanding of the different types of data that can be stored in a assert toks == [12092, 13, 309, 717, 247, 747, 17782, 281, 436, 12209, 285, 309, 717, 2820, 281, 755, 247, 1805, 4685, 273, 253, 1027, 3510, 273, 941, 326, 476, 320, 7141, 275, 247], "BAD OUTPUT!"

    explain in ascii art

  2. Assistant
    +--------------------------------------------------------------------------------------------------------+
    |                                                                                                        |
    |  This code implements a Mixture-of-Experts (MoE) Transformer model using the 'tinygrad' library.      |
    |  Here's a breakdown of the code's structure and functionality, visualized with ASCII art.             |
    |                                                                                                        |
    +--------------------------------------------------------------------------------------------------------+
    
    +-----------------------+
    | 1. Model Definition   |
    +-----------------------+
    
      +-------------------+   +-------------------+   +--------------------+
      |  Transformer Class  |-->|   MoE (MixtureFeedForward) Class |-->|  Linear layers within MoE |
      +-------------------+   +-------------------+   +--------------------+
          |
          | Contains layers (encoder, decoder), attention, normalization, MoE...
          |
          V
      +-----------------------------+
      |  Forward Pass (model call)   |
      +-----------------------------+
          |
          |  Processes input tokens through layers including the MoE layer.
          |
          V
      +-----------------------+
      |  Output (next token) |
      +-----------------------+
    
    +--------------------------------+
    | 2. Data Loading and Preparation |
    +--------------------------------+
    
      +---------------------+
      | Fetch Weights (HF)  |  Loads pre-trained weights from Hugging Face.
      +---------------------+       (Each "+" represents one safetensors file)
          |
          V
      +-----------------------+
      | State Dict Conversion | Converts HF weights into a dictionary of Tensor objects compatible with tinygrad.
      +-----------------------+
          |
          V
      +------------------------------------------------------+
      | Weight unpacking and distribution to model parameters |
      +------------------------------------------------------+
    
    +--------------------------------+
    | 3. Token Generation & Evaluation|
    +--------------------------------+
    
      +---------------------+
      |  Tokenizer Loading  |  Loads the tokenizer associated with the model.
      +---------------------+
          |
          V
      +-----------------------+
      | Initial Token Input  | Input starts with an initial token (e.g., 12092).
      +-----------------------+
          |
          V
      +----------------------------------------------------------------------------------------+
      | Loop for token generation (count=30):                                                   |
      |    - Forward pass through the model to predict the next token.                         |
      |    - Append the predicted token to the sequence.                                      |
      |    - Increment start position (for feeding back the sequence).                           |
      +----------------------------------------------------------------------------------------+
          |
          V
      +---------------------+
      | Output and Timing   | Decodes the generated tokens and prints timing information.
      +---------------------+
    
    +-----------------------------+
    |  Key Components Explained:   |
    +-----------------------------+
    
      - **Transformer**: The overall model architecture, containing layers for attention, normalization, and the MoE layer.
      - **MixtureFeedForward (MoE)**:  A feedforward layer that uses a mixture of experts.  It selects the top 'activated_experts' from 'num_experts' based on the input.
      - **Experts**: Multiple feedforward networks (within the MoE layer). The model learns to route different inputs to different experts.
      - **Gate**:  A linear layer that determines which experts to activate for a given input.  The output of the gate is softmaxed to obtain probabilities for each expert.
      - **Weights**: Pre-trained parameters of the model, loaded from Hugging Face.
      - **Tokenizer**: Used to convert text into tokens (numerical representations) and vice-versa.
    
    +-----------------------------------------------------------------------------------------------------+
    | Optimization Notes:                                                                                 |
    | - The code focuses on inference (token generation) rather than training.                         |
    | - Weights are loaded and unpacked efficiently for GPU usage.                                         |
    | - The code includes timing information to measure the performance of token generation.                  |
    | - Batch size is restricted to 1 for the MoE layer.                                               |
    +-----------------------------------------------------------------------------------------------------+
    

Responses were produced in different, uncontrolled contexts (system prompts, settings, prior turns). This is not a benchmark.