Skip to content
TheChatArchiveThe Chat Archive
Claudeclaude-opus-4-1-20250805· AnthropicHugging Face Arena Expert 5K

How would you implement positional encodings in such a model?

  1. User

    How would you implement positional encodings in such a model?

  2. Claude Assistant

    Here's how I would implement positional encodings in a bag-of-tokens model, focusing on maintaining computational efficiency:

    1. Basic Implementation Approach

    import torch
    import torch.nn as nn
    import math
    
    class PositionalBagOfTokens(nn.Module):
        def __init__(self, vocab_size, embed_dim, max_length=512):
            super().__init__()
            self.token_embeddings = nn.Embedding(vocab_size, embed_dim)
            self.positional_encodings = self.create_positional_encodings(max_length, embed_dim)
            
            # Optional: learnable weighting between token and position
            self.alpha = nn.Parameter(torch.tensor(0.5))
            
        def create_positional_encodings(self, max_length, embed_dim):
            """Create sinusoidal positional encodings"""
            pe = torch.zeros(max_length, embed_dim)
            position = torch.arange(0, max_length).unsqueeze(1).float()
            
            div_term = torch.exp(torch.arange(0, embed_dim, 2).float() * 
                                -(math.log(10000.0) / embed_dim))
            
            pe[:, 0::2] = torch.sin(position * div_term)
            pe[:, 1::2] = torch.cos(position * div_term)
            
            return nn.Parameter(pe, requires_grad=False)
        
        def forward(self, token_ids, positions=None):
            batch_size, seq_len = token_ids.shape
            
            # Get token embeddings
            token_embeds = self.token_embeddings(token_ids)  # [B, L, D]
            
            # Add positional information
            if positions is None:
                positions = torch.arange(seq_len, device=token_ids.device)
            
            pos_embeds = self.positional_encodings[positions]  # [L, D] or [B, L, D]
            
            # Combine token and positional embeddings
            combined = token_embeds + self.alpha * pos_embeds
            
            # Aggregate (mean pooling preserves position information better than sum)
            output = combined.mean(dim=1)  # [B, D]
            
            return output
    

    2. Position-Aware Aggregation Strategies

    Instead of simple mean/sum pooling, use position-aware aggregation:

    class PositionWeightedAggregation(nn.Module):
        def __init__(self, embed_dim, max_length=512):
            super().__init__()
            # Learn importance weights for different positions
            self.position_weights = nn.Parameter(torch.ones(max_length))
            self.embed_dim = embed_dim
            
        def forward(self, embeddings, mask=None):
            """
            embeddings: [B, L, D]
            mask: [B, L] binary mask for padding
            """
            batch_size, seq_len, _ = embeddings.shape
            
            # Get position weights for current sequence
            weights = self.position_weights[:seq_len].softmax(dim=0)  # [L]
            weights = weights.unsqueeze(0).unsqueeze(-1)  # [1, L, 1]
            
            if mask is not None:
                weights = weights * mask.unsqueeze(-1)
                weights = weights / (weights.sum(dim=1, keepdim=True) + 1e-8)
            
            # Weighted aggregation
            output = (embeddings * weights).sum(dim=1)  # [B, D]
            
            return output
    

    3. Relative Position Encodings (More Efficient)

    For bag-of-tokens, relative positions might be more useful:

    class RelativePositionalBagOfTokens(nn.Module):
        def __init__(self, vocab_size, embed_dim, window_size=16):
            super().__init__()
            self.token_embeddings = nn.Embedding(vocab_size, embed_dim)
            self.window_size = window_size
            
            # Learn embeddings for relative positions within a window
            self.relative_pos_embeddings = nn.Embedding(2 * window_size + 1, embed_dim)
            
            # Lightweight attention for local context
            self.context_proj = nn.Linear(embed_dim, embed_dim // 4)
            
        def forward(self, token_ids):
            batch_size, seq_len = token_ids.shape
            token_embeds = self.token_embeddings(token_ids)  # [B, L, D]
            
            # Add local positional context
            enhanced_embeds = []
            for i in range(seq_len):
                # Get window around position i
                start = max(0, i - self.window_size)
                end = min(seq_len, i + self.window_size + 1)
                
                # Compute relative positions
                rel_positions = torch.arange(start - i, end - i, device=token_ids.device)
                rel_positions = rel_positions + self.window_size  # Shift to positive indices
                
                # Get relative position embeddings
                rel_embeds = self.relative_pos_embeddings(rel_positions)  # [window, D]
                
                # Simple attention-like weighting based on position
                context = self.context_proj(token_embeds[:, start:end])  # [B, window, D//4]
                query = self.context_proj(token_embeds[:, i:i+1])  # [B, 1, D//4]
                
                weights = (query @ context.transpose(-2, -1)).softmax(dim=-1)  # [B, 1, window]
                
                # Weighted combination
                local_context = weights @ rel_embeds.unsqueeze(0)  # [B, 1, D]
                enhanced = token_embeds[:, i:i+1] + 0.1 * local_context
                enhanced_embeds.append(enhanced)
            
            enhanced_embeds = torch.cat(enhanced_embeds, dim=1)  # [B, L, D]
            
            # Final aggregation
            return enhanced_embeds.mean(dim=1)  # [B, D]
    

    4. Lightweight Factorized Approach

    For maximum efficiency, factorize the position encoding:

    class FactorizedPositionalBagOfTokens(nn.Module):
        def __init__(self, vocab_size, embed_dim, max_length=512, n_buckets=32):
            super().__init__()
            self.token_embeddings = nn.Embedding(vocab_size, embed_dim)
            self.n_buckets = n_buckets
            
            # Bucket positions to reduce memory
            self.position_buckets = nn.Embedding(n_buckets, embed_dim // 4)
            
            # Combine token and position information
            self.fusion = nn.Linear(embed_dim + embed_dim // 4, embed_dim)
            
        def get_position_bucket(self, positions, max_length):
            """Map positions to buckets (logarithmic or linear)"""
            # Linear bucketing for simplicity
            bucket_size = max_length // self.n_buckets
            return torch.clamp(positions // bucket_size, 0, self.n_buckets - 1)
        
        def forward(self, token_ids):
            batch_size, seq_len = token_ids.shape
            
            # Token embeddings
            token_embeds = self.token_embeddings(token_ids)  # [B, L, D]
            
            # Position buckets
            positions = torch.arange(seq_len, device=token_ids.device)
            buckets = self.get_position_bucket(positions, seq_len)
            pos_embeds = self.position_buckets(buckets)  # [L, D//4]
            
            # Concatenate and fuse
            pos_embeds = pos_embeds.unsqueeze(0).expand(batch_size, -1, -1)
            combined = torch.cat([token_embeds, pos_embeds], dim=-1)  # [B, L, D + D//4]
            
            fused = self.fusion(combined)  # [B, L, D]
            
            # Aggregate
            return fused.mean(dim=1)  # [B, D]
    

    Key Design Choices:

    1. Sinusoidal vs. Learned: Sinusoidal encodings require no training and generalize to any length, while learned encodings can adapt to your specific task.

    2. Addition vs. Concatenation: Adding positions to token embeddings is more parameter-efficient, while concatenation gives more flexibility.

    3. Aggregation Method: Mean pooling tends to preserve positional information better than sum pooling in practice.

    4. Relative vs. Absolute: Relative positions within local windows can be more informative and computationally cheaper than absolute positions.

    The factorized approach (#4) offers the best balance of expressivity and efficiency for most applications, with minimal overhead compared to vanilla bag-of-tokens.

Responses were produced in different, uncontrolled contexts (system prompts, settings, prior turns). This is not a benchmark.