How would you implement positional encodings in such a model?
- User
How would you implement positional encodings in such a model?
Assistant
Here's how I would implement positional encodings in a bag-of-tokens model, focusing on maintaining computational efficiency:
1. Basic Implementation Approach
import torch import torch.nn as nn import math class PositionalBagOfTokens(nn.Module): def __init__(self, vocab_size, embed_dim, max_length=512): super().__init__() self.token_embeddings = nn.Embedding(vocab_size, embed_dim) self.positional_encodings = self.create_positional_encodings(max_length, embed_dim) # Optional: learnable weighting between token and position self.alpha = nn.Parameter(torch.tensor(0.5)) def create_positional_encodings(self, max_length, embed_dim): """Create sinusoidal positional encodings""" pe = torch.zeros(max_length, embed_dim) position = torch.arange(0, max_length).unsqueeze(1).float() div_term = torch.exp(torch.arange(0, embed_dim, 2).float() * -(math.log(10000.0) / embed_dim)) pe[:, 0::2] = torch.sin(position * div_term) pe[:, 1::2] = torch.cos(position * div_term) return nn.Parameter(pe, requires_grad=False) def forward(self, token_ids, positions=None): batch_size, seq_len = token_ids.shape # Get token embeddings token_embeds = self.token_embeddings(token_ids) # [B, L, D] # Add positional information if positions is None: positions = torch.arange(seq_len, device=token_ids.device) pos_embeds = self.positional_encodings[positions] # [L, D] or [B, L, D] # Combine token and positional embeddings combined = token_embeds + self.alpha * pos_embeds # Aggregate (mean pooling preserves position information better than sum) output = combined.mean(dim=1) # [B, D] return output2. Position-Aware Aggregation Strategies
Instead of simple mean/sum pooling, use position-aware aggregation:
class PositionWeightedAggregation(nn.Module): def __init__(self, embed_dim, max_length=512): super().__init__() # Learn importance weights for different positions self.position_weights = nn.Parameter(torch.ones(max_length)) self.embed_dim = embed_dim def forward(self, embeddings, mask=None): """ embeddings: [B, L, D] mask: [B, L] binary mask for padding """ batch_size, seq_len, _ = embeddings.shape # Get position weights for current sequence weights = self.position_weights[:seq_len].softmax(dim=0) # [L] weights = weights.unsqueeze(0).unsqueeze(-1) # [1, L, 1] if mask is not None: weights = weights * mask.unsqueeze(-1) weights = weights / (weights.sum(dim=1, keepdim=True) + 1e-8) # Weighted aggregation output = (embeddings * weights).sum(dim=1) # [B, D] return output3. Relative Position Encodings (More Efficient)
For bag-of-tokens, relative positions might be more useful:
class RelativePositionalBagOfTokens(nn.Module): def __init__(self, vocab_size, embed_dim, window_size=16): super().__init__() self.token_embeddings = nn.Embedding(vocab_size, embed_dim) self.window_size = window_size # Learn embeddings for relative positions within a window self.relative_pos_embeddings = nn.Embedding(2 * window_size + 1, embed_dim) # Lightweight attention for local context self.context_proj = nn.Linear(embed_dim, embed_dim // 4) def forward(self, token_ids): batch_size, seq_len = token_ids.shape token_embeds = self.token_embeddings(token_ids) # [B, L, D] # Add local positional context enhanced_embeds = [] for i in range(seq_len): # Get window around position i start = max(0, i - self.window_size) end = min(seq_len, i + self.window_size + 1) # Compute relative positions rel_positions = torch.arange(start - i, end - i, device=token_ids.device) rel_positions = rel_positions + self.window_size # Shift to positive indices # Get relative position embeddings rel_embeds = self.relative_pos_embeddings(rel_positions) # [window, D] # Simple attention-like weighting based on position context = self.context_proj(token_embeds[:, start:end]) # [B, window, D//4] query = self.context_proj(token_embeds[:, i:i+1]) # [B, 1, D//4] weights = (query @ context.transpose(-2, -1)).softmax(dim=-1) # [B, 1, window] # Weighted combination local_context = weights @ rel_embeds.unsqueeze(0) # [B, 1, D] enhanced = token_embeds[:, i:i+1] + 0.1 * local_context enhanced_embeds.append(enhanced) enhanced_embeds = torch.cat(enhanced_embeds, dim=1) # [B, L, D] # Final aggregation return enhanced_embeds.mean(dim=1) # [B, D]4. Lightweight Factorized Approach
For maximum efficiency, factorize the position encoding:
class FactorizedPositionalBagOfTokens(nn.Module): def __init__(self, vocab_size, embed_dim, max_length=512, n_buckets=32): super().__init__() self.token_embeddings = nn.Embedding(vocab_size, embed_dim) self.n_buckets = n_buckets # Bucket positions to reduce memory self.position_buckets = nn.Embedding(n_buckets, embed_dim // 4) # Combine token and position information self.fusion = nn.Linear(embed_dim + embed_dim // 4, embed_dim) def get_position_bucket(self, positions, max_length): """Map positions to buckets (logarithmic or linear)""" # Linear bucketing for simplicity bucket_size = max_length // self.n_buckets return torch.clamp(positions // bucket_size, 0, self.n_buckets - 1) def forward(self, token_ids): batch_size, seq_len = token_ids.shape # Token embeddings token_embeds = self.token_embeddings(token_ids) # [B, L, D] # Position buckets positions = torch.arange(seq_len, device=token_ids.device) buckets = self.get_position_bucket(positions, seq_len) pos_embeds = self.position_buckets(buckets) # [L, D//4] # Concatenate and fuse pos_embeds = pos_embeds.unsqueeze(0).expand(batch_size, -1, -1) combined = torch.cat([token_embeds, pos_embeds], dim=-1) # [B, L, D + D//4] fused = self.fusion(combined) # [B, L, D] # Aggregate return fused.mean(dim=1) # [B, D]Key Design Choices:
-
Sinusoidal vs. Learned: Sinusoidal encodings require no training and generalize to any length, while learned encodings can adapt to your specific task.
-
Addition vs. Concatenation: Adding positions to token embeddings is more parameter-efficient, while concatenation gives more flexibility.
-
Aggregation Method: Mean pooling tends to preserve positional information better than sum pooling in practice.
-
Relative vs. Absolute: Relative positions within local windows can be more informative and computationally cheaper than absolute positions.
The factorized approach (#4) offers the best balance of expressivity and efficiency for most applications, with minimal overhead compared to vanilla bag-of-tokens.
-
Other models, same or similar prompt
Responses were produced in different, uncontrolled contexts (system prompts, settings, prior turns). This is not a benchmark.