https://arxiv.org/pdf/2409.02060 import time import numpy as np…
- User
https://arxiv.org/pdf/2409.02060
import time import numpy as np np.set_printoptions(suppress=True, linewidth=1000) import functools from tinygrad import Tensor, nn, Device, GlobalCounters from tinygrad.helpers import Timing, getenv from extra.models.llama import Transformer, convert_from_huggingface
class MixtureFeedForward: def init(self, num_experts:int, activated_experts:int, dim:int, hidden_dim:int, linear=nn.Linear): self.activated_experts = activated_experts self.gate = nn.Linear(dim, num_experts, bias=False) self.up_proj = Tensor.zeros(num_experts, hidden_dim, dim, dtype='bfloat16') self.down_proj = Tensor.zeros(num_experts, dim, hidden_dim, dtype='bfloat16') self.gate_proj = Tensor.zeros(num_experts, hidden_dim, dim, dtype='bfloat16') def call(self, x:Tensor) -> Tensor: assert x.shape[0] == 1, "only BS=1" assert x.shape[1] == 1, "only length=1" g = self.gate(x).float().softmax(-1)
g = g.squeeze() # (BS, length, num_experts) -> (num_experts,) probs, sel = g.topk(self.activated_experts) # run MoE x_up_gate = x.dot(self.gate_proj[sel].permute(0,2,1)).silu() * x.dot(self.up_proj[sel].permute(0,2,1)) x_down = x_up_gate.dot(self.down_proj[sel].permute(0,2,1)) return (x_down.float() * probs.reshape(self.activated_experts, 1, 1)).sum(axis=0)model is bf16, 1.3B active, 6.9B total
M3 Max is 400 GB/s, so 400/2.6 = ~154 tok/s
def fetch_weights() -> dict[str, Tensor]:
TODO: make this lazy so the 3 fetches can happen in parallel
m1 = Tensor.from_url("https://huggingface.co/allenai/OLMoE-1B-7B-0924/resolve/main/model-00001-of-00003.safetensors").to(Device.DEFAULT) m2 = Tensor.from_url("https://huggingface.co/allenai/OLMoE-1B-7B-0924/resolve/main/model-00002-of-00003.safetensors").to(Device.DEFAULT) m3 = Tensor.from_url("https://huggingface.co/allenai/OLMoE-1B-7B-0924/resolve/main/model-00003-of-00003.safetensors").to(Device.DEFAULT) return {**nn.state.safe_load(m1), **nn.state.safe_load(m2), **nn.state.safe_load(m3)}
if name == "main": if getenv("TORCH"): from transformers import OlmoeForCausalLM, AutoTokenizer model = OlmoeForCausalLM.from_pretrained("allenai/OLMoE-1B-7B-0924") tokenizer = AutoTokenizer.from_pretrained("allenai/OLMoE-1B-7B-0924") inputs = tokenizer("Hello", return_tensors="pt") generate_ids = model.generate(inputs.input_ids, max_length=30) out = tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0] print(out) exit(0)
with Timing("create model: "): model = Transformer(n_layers=16, dim=2048, hidden_dim=1024, n_heads=16, norm_eps=1e-5, qk_norm=1e-5, max_context=1024, vocab_size=50304, feed_forward=functools.partial(MixtureFeedForward, 64, 8)) model_state_dict = nn.state.get_state_dict(model) del model_state_dict['freqs_cis']
with Timing("load weights to GPU: "): nhf_state = convert_from_huggingface(fetch_weights(), model, 16, 16) # NOTE: i'm not sure this actually needs float32, it may just change the type of things downstream from it. but doesn't match torch w/o this for needs_float32 in ['tok_embeddings.weight']: nhf_state[needs_float32] = nhf_state[needs_float32].float() print(f"ram used: {GlobalCounters.mem_used/1e9:.2f} GB")
with Timing("unpack weights: "): nn.state.load_state_dict(model, nhf_state, verbose=False, strict=False, consume=True, realize=False) assert len(nhf_state) == 0 Tensor.realize(*list(nn.state.get_state_dict(model).values())) print(f"ram used: {GlobalCounters.mem_used/1e9:.2f} GB")
count = 30 temperature = 0
with Timing("load tokenizer: "): from transformers import AutoTokenizer tokenizer = AutoTokenizer.from_pretrained("allenai/OLMoE-1B-7B-0924")
toks = [12092] start_pos = 0 timings = [] for i in range(count): GlobalCounters.reset() st = time.perf_counter() tok = model(Tensor([toks[start_pos:]]), start_pos, temperature).item() timings.append(time.perf_counter()-st) toks.append(tok) start_pos += 1 print(toks) print(tokenizer.decode(toks)) print(f"fastest token {min(timings)*1e3:.2f} ms, {1/min(timings):.1f} tok/s")
if temperature == 0: # Hello, I am a newbie to this forum and I am trying to get a better understanding of the different types of data that can be stored in a assert toks == [12092, 13, 309, 717, 247, 747, 17782, 281, 436, 12209, 285, 309, 717, 2820, 281, 755, 247, 1805, 4685, 273, 253, 1027, 3510, 273, 941, 326, 476, 320, 7141, 275, 247], "BAD OUTPUT!"
explain in ascii art
Assistant
I'll provide an explanation of the given code in a structured and simple format, accompanied by some ASCII art to help illustrate the concepts.
Overview
The code is implementing a Mixture of Experts (MoE) language model, specifically the OLMoE-1B-7B-0924 model. It uses the
tinygradlibrary for building and running the model.Model Architecture
The model is based on a Transformer architecture with a Mixture of Experts feed-forward network.
+---------------+ | Input Token | +---------------+ | | v +---------------+ | Embedding | +---------------+ | | v +---------------+ | Transformer | | Layers | +---------------+ | | v +---------------+ | Mixture of | | Experts (MoE) | | Feed-Forward | +---------------+ | | v +---------------+ | Output | +---------------+Mixture of Experts (MoE)
The MoE is a type of feed-forward network that uses multiple experts, each with its own weights. The gate function determines which experts to use for a given input.
+---------------+ | Input | +---------------+ | | v +---------------+ | Gate Function| | (softmax) | +---------------+ | | v +---------------+ | Expert | | Selection | | (top-k) | +---------------+ | | v +---------------+ | Expert | | Networks | | (multiple) | +---------------+ | | v +---------------+ | Output | | (weighted sum)| +---------------+Code Explanation
- The
MixtureFeedForwardclass implements the MoE feed-forward network. - The
fetch_weightsfunction downloads the pre-trained model weights from Hugging Face. - The
Transformerclass is used to build the model architecture. - The model is loaded with the pre-trained weights and used for inference.
- The
AutoTokenizeris used to tokenize the input text.
Inference
The code performs inference on the input token "Hello" and generates output tokens.
- The input token is tokenized and passed through the model.
- The model generates output tokens one by one, using the previous token as input.
- The output tokens are decoded using the
AutoTokenizer.
The code measures the time taken for inference and reports the fastest token generation time and the average tokens per second.
- The
Other models, same or similar prompt
Responses were produced in different, uncontrolled contexts (system prompts, settings, prior turns). This is not a benchmark.