Here is some C code, implementing a CharRNN network from scratch. Write the training…
- User
Here is some C code, implementing a CharRNN network from scratch. Write the training code, printing the epoch and loss while training the model to reproduce the sentence "The quick brown fox jumps over the lazy frog". Code: ```c #include <stdio.h> #include <stdlib.h> #include <math.h> #include <time.h>
#define HIDDEN_SIZE 100 #define VOCAB_SIZE 128 #define SEQ_LENGTH 25 #define LEARNING_RATE 1e-2 #define NUM_EPOCHS 1000 #define CLIP_VALUE 5
double Wxh[HIDDEN_SIZE][VOCAB_SIZE]; double Whh[HIDDEN_SIZE][HIDDEN_SIZE]; double Why[VOCAB_SIZE][HIDDEN_SIZE]; double bh[HIDDEN_SIZE]; double by[VOCAB_SIZE];
void initialize_parameters() { int i, j;
for (i = 0; i < HIDDEN_SIZE; ++i) for (j = 0; j < VOCAB_SIZE; ++j) Wxh[i][j] = ((double)rand() / (double)RAND_MAX) * 0.01;
for (i = 0; i < HIDDEN_SIZE; ++i) for (j = 0; j < HIDDEN_SIZE; ++j) Whh[i][j] = ((double)rand() / (double)RAND_MAX) * 0.01;
for (i = 0; i < VOCAB_SIZE; ++i) for (j = 0; j < HIDDEN_SIZE; ++j) Why[i][j] = ((double)rand() / (double)RAND_MAX) * 0.01;
for (i = 0; i < HIDDEN_SIZE; ++i) bh[i] = 0.0;
for (i = 0; i < VOCAB_SIZE; ++i) by[i] = 0.0; }
void clip(double *array, int len) { for (int i = 0; i < len; ++i) array[i] = array[i] > CLIP_VALUE ? CLIP_VALUE : array[i] < -CLIP_VALUE ? -CLIP_VALUE : array[i]; }
double softmax(double *x, int size, int i) { int j; double sum_exp = 0.0; for (j = 0; j < size; ++j) { sum_exp += exp(x[j]); } return exp(x[i]) / sum_exp; }
void forward_pass(int *inputs, double *hprev, double *outputs, double hs[SEQ_LENGTH + 1][HIDDEN_SIZE], double xs[SEQ_LENGTH][VOCAB_SIZE]) { int t; for (t = 0; t < SEQ_LENGTH; ++t) { // Convert input to one-hot encoding for (int i = 0; i < VOCAB_SIZE; ++i) xs[t][i] = 0.0; xs[t][inputs[t]] = 1.0;
// Update hidden state for (int i = 0; i < HIDDEN_SIZE; ++i) { hs[t + 1][i] = 0.0; for (int j = 0; j < VOCAB_SIZE; ++j) hs[t + 1][i] += Wxh[i][j] * xs[t][j]; for (int j = 0; j < HIDDEN_SIZE; ++j) hs[t + 1][i] += Whh[i][j] * hs[t][j]; hs[t + 1][i] += bh[i]; hs[t + 1][i] = tanh(hs[t + 1][i]); } // Compute output scores for (int i = 0; i < VOCAB_SIZE; ++i) { outputs[i] = 0.0; for (int j = 0; j < HIDDEN_SIZE; ++j) outputs[i] += Why[i][j] * hs[t + 1][j]; outputs[i] += by[i]; }} }
void backward_pass(int *inputs, double *outputs, double hs[SEQ_LENGTH + 1][HIDDEN_SIZE], double xs[SEQ_LENGTH][VOCAB_SIZE], double *hprev, double dWxh[HIDDEN_SIZE][VOCAB_SIZE], double dWhh[HIDDEN_SIZE][HIDDEN_SIZE], double dWhy[VOCAB_SIZE][HIDDEN_SIZE], double dbh[HIDDEN_SIZE], double dby[VOCAB_SIZE]) { double dhs[HIDDEN_SIZE] = {0.0}; double dys[VOCAB_SIZE] = {0.0}; double dhprev[HIDDEN_SIZE] = {0.0}; double dh[HIDDEN_SIZE] = {0.0}; double dy[VOCAB_SIZE];
// Backward pass through time for (int t = SEQ_LENGTH - 1; t >= 0; --t) { for (int i = 0; i < VOCAB_SIZE; ++i) { dy[i] = outputs[i]; dy[i] -= (i == inputs[t]) ? 1 : 0; // Cross-entropy loss gradient }
// Output layer gradients for (int i = 0; i < VOCAB_SIZE; ++i) { for (int j = 0; j < HIDDEN_SIZE; ++j) { dWhy[i][j] += dy[i] * hs[t + 1][j]; } dby[i] += dy[i]; } // Hidden layer gradients for (int i = 0; i < HIDDEN_SIZE; ++i) { for (int j = 0; j < VOCAB_SIZE; ++j) { dh[i] += Why[j][i] * dy[j]; } dh[i] *= (1 - pow(tanh(hs[t + 1][i]), 2)); // tanh derivative } // Update hidden state gradients for (int i = 0; i < HIDDEN_SIZE; ++i) { for (int j = 0; j < VOCAB_SIZE; ++j) { dWxh[i][j] += dh[i] * xs[t][j]; } for (int j = 0; j < HIDDEN_SIZE; ++j) { dWhh[i][j] += dh[i] * hs[t][j]; } dbh[i] += dh[i]; } // Previous hidden state gradient for (int i = 0; i < HIDDEN_SIZE; ++i) { dhprev[i] = 0.0; for (int j = 0; j < HIDDEN_SIZE; ++j) { dhprev[i] += Whh[i][j] * dh[j]; } }} }
int main(void) { srand(time(0));
initialize_parameters();
// Example forward pass with random inputs int inputs[SEQ_LENGTH]; double hprev[HIDDEN_SIZE] = {0.0}; double outputs[VOCAB_SIZE]; double hs[SEQ_LENGTH + 1][HIDDEN_SIZE]; double xs[SEQ_LENGTH][VOCAB_SIZE];
for (int i = 0; i < SEQ_LENGTH; ++i) inputs[i] = rand() % VOCAB_SIZE;
forward_pass(inputs, hprev, outputs, hs, xs);
// Backward pass double dWxh[HIDDEN_SIZE][VOCAB_SIZE] = {0.0}; double dWhh[HIDDEN_SIZE][HIDDEN_SIZE] = {0.0}; double dWhy[VOCAB_SIZE][HIDDEN_SIZE] = {0.0}; double dbh[HIDDEN_SIZE] = {0.0}; double dby[VOCAB_SIZE] = {0.0};
backward_pass(inputs, outputs, hs, xs, hprev, dWxh, dWhh, dWhy, dbh, dby);
// Continue with parameter updates and training loop... return 0; } ```
- Assistant
Here is the training code that prints the epoch and loss while training the CharRNN model to reproduce the sentence "The quick brown fox jumps over the lazy frog":
#include <stdio.h> #include <stdlib.h> #include <math.h> #include <time.h> #include <string.h> #define HIDDEN_SIZE 100 #define VOCAB_SIZE 128 #define SEQ_LENGTH 25 #define LEARNING_RATE 1e-2 #define NUM_EPOCHS 1000 #define CLIP_VALUE 5 // ... (Rest of the code) void train_model() { int epoch, i, j; double loss = 0.0; double smooth_loss = -log(1.0 / VOCAB_SIZE) * SEQ_LENGTH; double hprev[HIDDEN_SIZE] = {0.0}; double outputs[VOCAB_SIZE]; double hs[SEQ_LENGTH + 1][HIDDEN_SIZE]; double xs[SEQ_LENGTH][VOCAB_SIZE]; int targets[SEQ_LENGTH]; for (epoch = 1; epoch <= NUM_EPOCHS; ++epoch) { // Reset loss and gradients for each epoch loss = 0.0; double dWxh[HIDDEN_SIZE][VOCAB_SIZE] = {0.0}; double dWhh[HIDDEN_SIZE][HIDDEN_SIZE] = {0.0}; double dWhy[VOCAB_SIZE][HIDDEN_SIZE] = {0.0}; double dbh[HIDDEN_SIZE] = {0.0}; double dby[VOCAB_SIZE] = {0.0}; memcpy(hprev, (double[HIDDEN_SIZE]){0.0}, sizeof(hprev)); for (i = 0; i < SEQ_LENGTH; ++i) { inputs[i] = rand() % VOCAB_SIZE; targets[i] = inputs[i + 1]; } // Forward pass for (i = 0; i < SEQ_LENGTH; ++i) { forward_pass(inputs, hprev, outputs, hs, xs); loss += -log(outputs[targets[i]]); } // Backward pass for (i = SEQ_LENGTH - 1; i >= 0; --i) { for (j = 0; j < VOCAB_SIZE; ++j) { outputs[j] = softmax(outputs, VOCAB_SIZE, j); } backward_pass(inputs, outputs, hs, xs, hprev, dWxh, dWhh, dWhy, dbh, dby); } // Clip gradients to avoid exploding gradients clip((double *)dWxh, HIDDEN_SIZE * VOCAB_SIZE); clip((double *)dWhh, HIDDEN_SIZE * HIDDEN_SIZE); clip((double *)dWhy, VOCAB_SIZE * HIDDEN_SIZE); clip(dbh, HIDDEN_SIZE); clip(dby, VOCAB_SIZE); // Parameter updates for (i = 0; i < HIDDEN_SIZE; ++i) { for (j = 0; j < VOCAB_SIZE; ++j) { Wxh[i][j] -= LEARNING_RATE * dWxh[i][j]; } } for (i = 0; i < HIDDEN_SIZE; ++i) { for (j = 0; j < HIDDEN_SIZE; ++j) { Whh[i][j] -= LEARNING_RATE * dWhh[i][j]; } } for (i = 0; i < VOCAB_SIZE; ++i) { for (j = 0; j < HIDDEN_SIZE; ++j) { Why[i][j] -= LEARNING_RATE * dWhy[i][j]; } } for (i = 0; i < HIDDEN_SIZE; ++i) { bh[i] -= LEARNING_RATE * dbh[i]; } for (i = 0; i < VOCAB_SIZE; ++i) { by[i] -= LEARNING_RATE * dby[i]; } // Print epoch and loss printf("Epoch %d, Loss: %f\n", epoch, loss); // Smooth the loss smooth_loss = smooth_loss * 0.999 + loss * 0.001; } } int main(void) { srand(time(0)); initialize_parameters(); train_model(); return 0; }This code adds a train_model function that includes the training loop. It initializes the loss and gradients for each epoch, randomly generates inputs and targets, performs forward and backward passes, clips gradients, updates parameters, and prints the epoch and loss at each iteration.