how deos this ai work import os import sys import random import argparse import torch…
- User
how deos this ai work import os import sys import random import argparse
import torch import numpy as np import torch.nn.functional as F
from torch.utils.data._utils.collate import default_collate from torchvision import transforms from datasets import load_dataset from transformers import AutoTokenizer, AutoModel from diffusers import DDPMScheduler from diffusers import DDPMPipeline from diffusers.optimization import get_cosine_schedule_with_warmup from accelerate import Accelerator from tqdm.auto import tqdm
sys.path.insert(0, '%s'%os.path.join(os.path.dirname(file), '../src/')) from markup2im_constants import get_image_size, get_input_field, get_encoder_model_type, get_color_mode from markup2im_models import create_image_decoder, encode_text, save_model
torch.backends.cuda.matmul.allow_tf32=True # for speed
def process_args(args): parser = argparse.ArgumentParser(description='Train a diffusion model.')
parser.add_argument('--dataset_name', type=str, default='yuntian-deng/im2latex-100k', help=('Specifies which dataset to use.' )) parser.add_argument('--input_field', type=str, default=None, help=('Field in the dataset containing input markups. If set to None, will be inferred according to dataset_name.' )) parser.add_argument('--color_mode', type=str, default=None, help=('Specifies grayscale (grayscale) or RGB (rgb). If set to None, will be inferred according to dataset_name.' )) parser.add_argument('--encoder_model_type', type=str, default=None, help=('Specifies encoder model type. If set to None, will be inferred according to dataset_name.' )) parser.add_argument('--image_height', type=int, default=None, help=('Specifies the height of images to generate. If set to None, will be inferred according to dataset_name.' )) parser.add_argument('--image_width', type=int, default=None, help=('Specifies the width of images to generate. If set to None, will be inferred according to dataset_name.' )) parser.add_argument('--save_dir', type=str, required=True, help=('Output directory for saving model checkpoints.' )) parser.add_argument('--split', type=str, default='train', help=('Dataset split.' )) parser.add_argument('--batch_size', type=int, default=16, help=('Batch size.' )) parser.add_argument('--gradient_accumulation_steps', type=int, default=1, help=('Gradient accumulation steps.' )) parser.add_argument('--num_epochs', type=int, default=100, help=('Number of epochs.' )) parser.add_argument('--scheduled_sampling_weights_start', type=float, nargs='+', default=[0,], help=("""The starting weight of applying scheduled sampling. Pass in a list for higher-order scheduled sampling (higher $m$ where $m$ denotes the number of rollout steps, see our paper for more details). For example, `--scheduled_sampling_weights_start 0 --scheduled_sampling_weights_start 0.5` will start from 0 weight of applying first-order (m=1) scheduled sampling and 0.5 of applying second-order (m=2) scheduled sampling.""" )) parser.add_argument('--scheduled_sampling_weights_end', type=float, nargs='+', default=[0.8,], help=("""The end weight of applying scheduled sampling. Pass in a list for higher-order (higher number of rollout steps) scheduled sampling. For example, `--scheduled_sampling_weights_end 0.2 --scheduled_sampling_weights_end 0.3` will end with 0.2 weight of applying first-order (m=1) scheduled sampling and 0.3 of applying second-order (m=2) scheduled sampling.""" )) parser.add_argument('--learning_rate', type=int, default=1e-4, help=('Learning rate.' )) parser.add_argument('--lr_warmup_steps', type=int, default=500, help=('Lr warmup steps.' )) parser.add_argument('--clip_grad_norm', type=float, default=1.0, help=('Clip gradient norm.' )) parser.add_argument('--save_model_every', type=int, default=5, help=('Saves intermediate model checkpoints every this many steps.' )) parser.add_argument('--mixed_precision', type=str, default='no', help=('Can be fp16 or no (fp32).' )) parser.add_argument('--max_input_length', type=int, default=1024, help=('Max input length. Longer inputs will be truncated.' )) parser.add_argument('--num_dataloader_workers', type=int, default=1, help=('Number of workers for dataloder.' )) parser.add_argument('--seed1', type=int, default=42, help=('Random seed for shuffling data. Shouldn\'t be changed to be comparable to numbers reported in the paper.' )) parser.add_argument('--seed2', type=int, default=1234, help=('Random seed for data loader. Shouldn\'t be changed to be comparable to numbers reported in the paper.' )) parameters = parser.parse_args(args) return parametersdef train(train_dataloader, save_dir, save_model_every,
text_encoder, image_decoder, noise_scheduler,
scheduled_sampling_weights_start, scheduled_sampling_weights_end,
optimizer, lr_scheduler, num_epochs, gradient_accumulation_steps=1,
clip_grad_norm=1.0, learning_rate=1e-4,
mixed_precision='no'): learning_rate = optimizer.defaults['lr'] # Initialize accelerator and tensorboard logging accelerator = Accelerator( mixed_precision=mixed_precision, gradient_accumulation_steps=gradient_accumulation_steps, logging_dir=os.path.join(save_dir, "logs") ) text_encoder = text_encoder.to(accelerator.device) if accelerator.is_main_process: accelerator.init_trackers("markup2im_train")image_decoder, optimizer, train_dataloader, lr_scheduler = accelerator.prepare( image_decoder, optimizer, train_dataloader, lr_scheduler ) scheduled_sampling_weights_start = np.array(scheduled_sampling_weights_start) scheduled_sampling_weights_end = np.array(scheduled_sampling_weights_end) global_step = 0 total_steps = len(train_dataloader)*num_epochs for epoch in range(num_epochs): progress_bar = tqdm(total=len(train_dataloader), disable=not accelerator.is_local_main_process) progress_bar.set_description(f"Epoch {epoch}") for step, batch in enumerate(train_dataloader): # the weights of scheduled sampling change linearly throughout training m_probs = scheduled_sampling_weights_start + global_step / total_steps * \ (scheduled_sampling_weights_end - scheduled_sampling_weights_start) m_probs = np.insert(m_probs, 0, 1-m_probs.sum()) # the weight of not applying scheduled sampling (m=0) disp_str = '"' + ' '.join([f'm={i} ({m_probs[i]:.2f})' for i in range(len(m_probs))]) + '"' clean_images = batch['images'].to(accelerator.device) input_ids = batch['input_ids'].to(accelerator.device) masks = batch['attention_mask'].to(accelerator.device) encoder_hidden_states = encode_text(text_encoder, input_ids, masks) bs = clean_images.shape[0] # Sample a random timestep for each image timesteps = torch.randint(0, noise_scheduler.num_train_timesteps, (bs,), device=clean_images.device).long() # Sample m in scheduled sampling according to m_probs # Note that we use the same m per batch on the same device for efficiency m = np.random.choice(len(m_probs), size=1, p=m_probs)[0] # If there's not enough number of steps then decrease m while max(timesteps) >= noise_scheduler.num_train_timesteps-m: m -= 1 # find input to the diffusion model with torch.no_grad(): # Sample noise to add to the images noise = torch.randn(clean_images.shape).to(clean_images.device) # first, sample t + m using Q noisy_images_t_plus_m = noise_scheduler.add_noise(clean_images, noise, timesteps+m) noisy_images_t_plus_s = noisy_images_t_plus_m # next, roll back to t using P for s in range(m): # predict noise noise_pred_rollback_s = image_decoder(noisy_images_t_plus_s, timesteps+m-s, encoder_hidden_states, attention_mask=masks)["sample"] lambs_s, alpha_prod_ts_s = noise_scheduler.get_lambda_and_alpha(timesteps+m-s) # clean img predicted x_0_pred = (noisy_images_t_plus_s - lambs_s.view(-1, 1, 1, 1) * noise_pred_rollback_s) / alpha_prod_ts_s.view(-1, 1, 1, 1) noise = torch.randn(clean_images.shape).to(clean_images.device) # get previous step sample noisy_images_t_plus_s_minus_one = noise_scheduler.add_noise(x_0_pred, noise, timesteps + m-s-1) # update noisy_images_t_plus_s = noisy_images_t_plus_s_minus_one noisy_images_t = noisy_images_t_plus_s with accelerator.accumulate(image_decoder): # Predict the noise residual noise_pred = image_decoder(noisy_images_t, timesteps, encoder_hidden_states, attention_mask=masks)["sample"] lambs_t, alpha_prod_ts_t = noise_scheduler.get_lambda_and_alpha(timesteps) noise = (noisy_images_t - alpha_prod_ts_t.view(-1, 1, 1, 1) * clean_images) / lambs_t.view(-1, 1, 1, 1) loss = F.mse_loss(noise_pred, noise) accelerator.backward(loss) accelerator.clip_grad_norm_(image_decoder.parameters(), clip_grad_norm) optimizer.step() lr_scheduler.step() optimizer.zero_grad() progress_bar.update(1) logs = {"loss": loss.detach().item()*gradient_accumulation_steps, "lr": lr_scheduler.get_last_lr()[0], "step": global_step, 'scheduled sampling': disp_str} progress_bar.set_postfix(**logs) accelerator.log(logs, step=global_step) global_step += 1 if epoch % save_model_every == 0: save_model(image_decoder, os.path.join(save_dir, f'model_e{num_epochs}_lr{learning_rate}.pt.{epoch}'))def main(args): # Check arguments assert len(args.scheduled_sampling_weights_start) == len(args.scheduled_sampling_weights_end) assert all([0 <= item <= 1 for item in args.scheduled_sampling_weights_start]) assert all([0 <= item <= 1 for item in args.scheduled_sampling_weights_end]) assert sum(args.scheduled_sampling_weights_start) <= 1 assert sum(args.scheduled_sampling_weights_end) <= 1 # Get default arguments if (args.image_height is not None) and (args.image_width is not None): image_size = (args.image_height, args.image_width) else: print (f'Using default image size for dataset {args.dataset_name}') image_size = get_image_size(args.dataset_name) print (f'Default image size: {image_size}') args.image_size = image_size if args.input_field is not None: input_field = args.input_field else: print (f'Using default input field for dataset {args.dataset_name}') input_field = get_input_field(args.dataset_name) print (f'Default input field: {input_field}') args.input_field = input_field if args.encoder_model_type is not None: encoder_model_type = args.encoder_model_type else: print (f'Using default encoder model type for dataset {args.dataset_name}') encoder_model_type = get_encoder_model_type(args.dataset_name) print (f'Default encoder model type: {encoder_model_type}') args.encoder_model_type = encoder_model_type if args.color_mode is not None: color_mode = args.color_mode else: print (f'Using default color mode for dataset {args.dataset_name}') color_mode = get_color_mode(args.dataset_name) print (f'Default color mode: {color_mode}') args.color_mode = color_mode assert args.color_mode in ['grayscale', 'rgb'] if args.color_mode == 'grayscale': args.color_channels = 1 else: args.color_channels = 3
# Load data dataset = load_dataset(args.dataset_name, split=args.split) dataset = dataset.shuffle(seed=args.seed1) # Load input tokenizer tokenizer = AutoTokenizer.from_pretrained(args.encoder_model_type) # Load input encoder text_encoder = AutoModel.from_pretrained(args.encoder_model_type).cuda() # Preprocess data to form batches transform_list = [] if args.color_mode == 'grayscale': transform_list.append(transforms.Grayscale(num_output_channels=args.color_channels)) preprocess_image = transforms.Compose( transform_list + [ transforms.ToTensor(), transforms.Normalize([0.5], [0.5]), ] ) def preprocess_formula(formula): example = tokenizer(formula, truncation=True, max_length=args.max_input_length) input_ids = example['input_ids'] attention_mask = example['attention_mask'] return input_ids, attention_mask def transform(examples): images = [preprocess_image(image.convert("RGB")) for image in examples["image"]] gold_images = [image for image in examples["image"]] formulas_and_masks = [preprocess_formula(formula) for formula in examples[args.input_field]] formulas = [item[0] for item in formulas_and_masks] masks = [item[1] for item in formulas_and_masks] filenames = examples['filename'] return {'images': images, 'input_ids': formulas, 'attention_mask': masks, 'filenames': filenames, 'gold_images': gold_images} dataset.set_transform(transform) def collate_fn(examples): eos_id = tokenizer.encode(tokenizer.eos_token)[0] # legacy code, might be unnecessary max_len = max([len(example['input_ids']) for example in examples]) + 1 examples_out = [] for example in examples: example_out = {} orig_len = len(example['input_ids']) formula = example['input_ids'] + [eos_id,] * (max_len - orig_len) example_out['input_ids'] = torch.LongTensor(formula) attention_mask = example['attention_mask'] + [1,] + [0,] * (max_len - orig_len - 1) example_out['attention_mask'] = torch.LongTensor(attention_mask) example_out['images'] = example['images'] examples_out.append(example_out) batch = default_collate(examples_out) filenames = [example['filenames'] for example in examples] gold_images = [example['gold_images'] for example in examples] batch['filenames'] = filenames batch['gold_images'] = gold_images return batch torch.manual_seed(args.seed2) random.seed(args.seed2) np.random.seed(args.seed2) train_dataloader = torch.utils.data.DataLoader(dataset, batch_size=args.batch_size, \ shuffle=True, collate_fn=collate_fn, worker_init_fn=np.random.seed(0), \ num_workers=args.num_dataloader_workers) # Create and load models text_encoder = AutoModel.from_pretrained(args.encoder_model_type).cuda() # forward a fake batch to figure out cross_attention_dim hidden_states = encode_text(text_encoder, torch.zeros(1,1).long().cuda(), None) cross_attention_dim = hidden_states.shape[-1] image_decoder = create_image_decoder(image_size=args.image_size, color_channels=args.color_channels, \ cross_attention_dim=cross_attention_dim) image_decoder = image_decoder.cuda() noise_scheduler = DDPMScheduler(num_train_timesteps=1000, tensor_format="pt") # Optimization optimizer = torch.optim.AdamW(image_decoder.parameters(), lr=args.learning_rate) lr_scheduler = get_cosine_schedule_with_warmup( optimizer=optimizer, num_warmup_steps=args.lr_warmup_steps, num_training_steps=(len(train_dataloader) * args.num_epochs), ) train(train_dataloader, args.save_dir, args.save_model_every, \ text_encoder, image_decoder, noise_scheduler, \ args.scheduled_sampling_weights_start, args.scheduled_sampling_weights_end, \ optimizer, lr_scheduler, args.num_epochs, \ gradient_accumulation_steps=args.gradient_accumulation_steps, \ clip_grad_norm=args.clip_grad_norm, \ mixed_precision=args.mixed_precision) # Save final model save_model(image_decoder, os.path.join(args.save_dir, f'model_e{args.num_epochs}_lr{args.learning_rate}.pt.{args.num_epochs}'))if name == 'main': args = process_args(sys.argv[1:]) main(args)
- Assistant
The code you've provided is a Python script for training a diffusion model using PyTorch and HuggingFace's
transformersanddiffuserslibraries. This model is designed to generate images from textual input, leveraging techniques like scheduled sampling and conditional diffusion.Here's a detailed breakdown of how this AI model works:
1. Overview
The script is intended to train a diffusion model that can generate images conditioned on text inputs. It uses scheduled sampling to improve training stability and performance. The primary components of the script include:
- Data Loading and Preprocessing: Loading a dataset and preprocessing both the text and image data.
- Model Components:
- Text Encoder: To encode textual inputs into embeddings.
- Image Decoder (Diffusion Model): To generate images based on the encoded text.
- Training Loop: Implements the training procedure with scheduled sampling.
- Utilities and Configurations: Argument parsing, logging, and saving checkpoints.
2. Key Components Explained
Imports and Setup
import os import sys import random import argparse import torch import numpy as np import torch.nn.functional as F from torch.utils.data._utils.collate import default_collate from torchvision import transforms from datasets import load_dataset from transformers import AutoTokenizer, AutoModel from diffusers import DDPMScheduler from diffusers import DDPMPipeline from diffusers.optimization import get_cosine_schedule_with_warmup from accelerate import Accelerator from tqdm.auto import tqdm- PyTorch: For building and training neural networks.
- HuggingFace Transformers: To utilize pre-trained text encoders.
- HuggingFace Datasets: For loading and managing datasets.
- Diffusers: A library for diffusion models.
- Accelerate: For handling multi-GPU training and mixed precision.
- tqdm: For progress bars.
Custom Modules
sys.path.insert(0, '%s'%os.path.join(os.path.dirname(__file__), '../src/')) from markup2im_constants import get_image_size, get_input_field, get_encoder_model_type, get_color_mode from markup2im_models import create_image_decoder, encode_text, save_model- markup2im_constants: Contains dataset-specific constants and helper functions.
- markup2im_models: Defines the model architecture and utility functions for the image decoder and text encoder.
Argument Parsing
The
process_argsfunction defines and parses command-line arguments, allowing customization of various training parameters like dataset name, batch size, learning rate, etc.Data Loading and Preprocessing
Dataset Loading
dataset = load_dataset(args.dataset_name, split=args.split) dataset = dataset.shuffle(seed=args.seed1)- Loads the specified dataset (e.g., 'yuntian-deng/im2latex-100k') and shuffles it.
Tokenization and Text Encoding
tokenizer = AutoTokenizer.from_pretrained(args.encoder_model_type) text_encoder = AutoModel.from_pretrained(args.encoder_model_type).cuda()- Uses a pre-trained tokenizer and model from the specified
encoder_model_type(e.g., 'bert-base-uncased').
Image Preprocessing
transform_list = [] if args.color_mode == 'grayscale': transform_list.append(transforms.Grayscale(num_output_channels=args.color_channels)) preprocess_image = transforms.Compose( transform_list + [ transforms.ToTensor(), transforms.Normalize([0.5], [0.5]), ] )- Converts images to grayscale if specified.
- Normalizes images to have pixel values in the range [-1, 1].
Dataset Transformation Function
def transform(examples): images = [preprocess_image(image.convert("RGB")) for image in examples["image"]] ... return {'images': images, 'input_ids': formulas, 'attention_mask': masks, 'filenames': filenames, 'gold_images': gold_images}- Applies the preprocessing to images and tokenizes the text inputs.
- Returns a dictionary containing processed images, input IDs, attention masks, filenames, and original images.
Collate Function
def collate_fn(examples): ... return batch- Pads sequences to the maximum length in the batch.
- Prepares the batch for the DataLoader.
Model Components
Text Encoder
text_encoder = AutoModel.from_pretrained(args.encoder_model_type).cuda()- Encodes textual input into embeddings that will condition the image decoder.
- The embeddings (hidden states) are used as conditioning information.
Image Decoder (Diffusion Model)
image_decoder = create_image_decoder(image_size=args.image_size, color_channels=args.color_channels, cross_attention_dim=cross_attention_dim) image_decoder = image_decoder.cuda()- The image decoder is a U-Net-based architecture commonly used in diffusion models.
- It takes noisy images and timesteps as input, using cross-attention with the text embeddings to generate denoised images.
Noise Scheduler
noise_scheduler = DDPMScheduler(num_train_timesteps=1000, tensor_format="pt")- Manages the noise schedule for the diffusion process.
- Defines how noise is added during training and how the model learns to denoise.
Training Function
Accelerator Setup
accelerator = Accelerator( mixed_precision=mixed_precision, gradient_accumulation_steps=gradient_accumulation_steps, logging_dir=os.path.join(save_dir, "logs") )Acceleratorhandles device placement, mixed precision, and distributed training.- Initializes logging for tracking training progress.
Scheduled Sampling
Scheduled sampling is a technique used to mitigate exposure bias during training, especially in sequence generation tasks. It involves feeding the model's own predictions back into itself during training with a certain probability.
In the context of this diffusion model:
- Higher-order Scheduled Sampling: The model rolls forward
msteps into the future and then rolls back to the current timesteptusing its own predictions. - Weight Adjustment: The probabilities of applying scheduled sampling (
m_probs) adjust linearly from their starting values (scheduled_sampling_weights_start) to their ending values (scheduled_sampling_weights_end) over the course of training.
Training Loop
for epoch in range(num_epochs): ... for step, batch in enumerate(train_dataloader): ... # Sample m in scheduled sampling according to m_probs m = np.random.choice(len(m_probs), size=1, p=m_probs)[0] ... # Roll back to t using P (the model's own predictions) for s in range(m): ... # Forward pass ... # Loss computation loss = F.mse_loss(noise_pred, noise) ...- Timesteps Sampling: Randomly selects timesteps
tfor each image in the batch. - Scheduled Sampling Application: Samples an
mvalue to determine how many steps to roll out in the future. - Noise Addition and Removal:
- Forward Process: Adds noise to the images at timestep
t + m. - Reverse Process: Uses the model to predict and remove noise step by step back to timestep
t.
- Forward Process: Adds noise to the images at timestep
- Loss Computation: Mean squared error between the predicted noise and the true noise.
Optimization and Logging
accelerator.backward(loss) optimizer.step() lr_scheduler.step() optimizer.zero_grad()- Gradient Accumulation: Accumulates gradients over several steps if
gradient_accumulation_steps > 1. - Gradient Clipping: Clamps gradients to prevent exploding gradients.
Model Saving
if epoch % save_model_every == 0: save_model(image_decoder, os.path.join(save_dir, f'model_e{num_epochs}_lr{learning_rate}.pt.{epoch}'))- Periodically saves the model checkpoints for later use or evaluation.
3. How the Diffusion Model Works
Diffusion Models Overview
Diffusion models are generative models that learn to generate data by reversing a gradual noising process. They work by:
- Forward Process (Diffusion): Gradually adding noise to data over several timesteps until the data becomes pure noise.
- Reverse Process (Denoising): Learning to reverse this process by progressively denoising from pure noise back to the original data.
Conditional Diffusion with Text
In this script, the diffusion model is conditional—it generates images based on text input.
- Text Conditioning: The text encoder produces embeddings that condition the image generation process.
- Cross-Attention: The image decoder employs cross-attention mechanisms to incorporate text information into the image generation.
Scheduled Sampling in Training
Scheduled sampling introduces the model's own predictions into the training process:
- Purpose: To bridge the gap between training and inference behaviors.
- Implementation:
- At each training step, the model decides whether to use the ground truth data or its own previous predictions to generate the next input.
- This decision is controlled by a probability that changes over time (from
scheduled_sampling_weights_starttoscheduled_sampling_weights_end).
Training Objective
- The model aims to predict the noise added to the images at each timestep.
- Minimizing the difference between the predicted noise and the actual noise helps the model learn the reverse denoising process.
4. Execution Flow
- Argument Parsing: The script begins by parsing command-line arguments to get training configurations.
- Data Loading: Loads and preprocesses both text and image data.
- Model Initialization: Sets up the text encoder, image decoder, and noise scheduler.
- Optimizer and Scheduler Setup: Configures the optimizer and learning rate scheduler.
- Training Loop:
- Iterates over epochs and batches.
- Applies scheduled sampling.
- Performs the forward and reverse diffusion processes.
- Computes loss and updates model parameters.
- Logs progress and saves checkpoints.
5. Additional Details
- Mixed Precision Training: If specified, the model uses half-precision floats (
fp16) to reduce memory usage and potentially increase computational speed. - Gradient Accumulation: Allows effective batch sizes larger than what can fit in memory by accumulating gradients over several batches.
- Learning Rate Scheduling: Uses a cosine schedule with warmup to adjust the learning rate during training.
6. Conclusion
This AI model leverages diffusion processes to generate images conditioned on text. By integrating a text encoder and an image decoder within a diffusion framework, the model learns to produce images that correspond to textual descriptions. Scheduled sampling enhances the training by exposing the model to its own predictions, thereby improving its robustness during inference.
If you have any specific questions about certain parts of the code or concepts, feel free to ask!