Untitled conversation
- User
#include <stdio.h> #include <stdlib.h> #include <math.h> #include <time.h>”> #define HIDDEN_SIZE 100 #define VOCAB_SIZE 128 #define SEQ_LENGTH 25 #define LEARNING_RATE 1e-2 #define NUM_EPOCHS 1000 #define CLIP_VALUE 5 double Wxh[HIDDEN_SIZE][VOCAB_SIZE]; double Whh[HIDDEN_SIZE][HIDDEN_SIZE]; double Why[VOCAB_SIZE][HIDDEN_SIZE]; double bh[HIDDEN_SIZE]; double by[VOCAB_SIZE]; void initialize_parameters() { int i, j; for (i = 0; i < HIDDEN_SIZE; ++i) for (j = 0; j < VOCAB_SIZE; ++j) Wxh[i][j] = ((double)rand() / (double)RAND_MAX) * 0.01; for (i = 0; i < HIDDEN_SIZE; ++i) for (j = 0; j < HIDDEN_SIZE; ++j) Whh[i][j] = ((double)rand() / (double)RAND_MAX) * 0.01; for (i = 0; i < VOCAB_SIZE; ++i) for (j = 0; j < HIDDEN_SIZE; ++j) Why[i][j] = ((double)rand() / (double)RAND_MAX) * 0.01; for (i = 0; i < HIDDEN_SIZE; ++i) bh[i] = 0.0; for (i = 0; i < VOCAB_SIZE; ++i) by[i] = 0.0; } void clip(double *array, int len) { for (int i = 0; i < len; ++i) array[i] = array[i] > CLIP_VALUE ? CLIP_VALUE : array[i] < -CLIP_VALUE ? -CLIP_VALUE : array[i]; } double softmax(double *x, int size, int i) { int j; double sum_exp = 0.0; for (j = 0; j < size; ++j) { sum_exp += exp(x[j]); } return exp(x[i]) / sum_exp; } void one_hot_encode(char character, double *encoded) { int index = character; for (int i = 0; i < VOCAB_SIZE; ++i) { encoded[i] = (i == index) ? 1.0 : 0.0; } } char one_hot_decode(double *encoded) { int index = 0; double max_val = encoded[0]; for (int i = 1; i < VOCAB_SIZE; ++i) { if (encoded[i] > max_val) { index = i; max_val = encoded[i]; } } return (char)index; }Given the above C code, continue the implementation of a CharRNN
- Assistant
#include <stdio.h> #include <stdlib.h> #include <math.h> #include <time.h> #define HIDDEN_SIZE 100 #define VOCAB_SIZE 128 #define SEQ_LENGTH 25 #define LEARNING_RATE 1e-2 #define NUM_EPOCHS 1000 #define CLIP_VALUE 5 double Wxh[HIDDEN_SIZE][VOCAB_SIZE]; double Whh[HIDDEN_SIZE][HIDDEN_SIZE]; double Why[VOCAB_SIZE][HIDDEN_SIZE]; double bh[HIDDEN_SIZE]; double by[VOCAB_SIZE]; void initialize_parameters() { int i, j; // Initialize Wxh for (i = 0; i < HIDDEN_SIZE; ++i) { for (j = 0; j < VOCAB_SIZE; ++j) { Wxh[i][j] = ((double)rand() / (double)RAND_MAX) * 0.01; } } // Initialize Whh for (i = 0; i < HIDDEN_SIZE; ++i) { for (j = 0; j < HIDDEN_SIZE; ++j) { Whh[i][j] = ((double)rand() / (double)RAND_MAX) * 0.01; } } // Initialize Why for (i = 0; i < VOCAB_SIZE; ++i) { for (j = 0; j < HIDDEN_SIZE; ++j) { Why[i][j] = ((double)rand() / (double)RAND_MAX) * 0.01; } } // Initialize bh for (i = 0; i < HIDDEN_SIZE; ++i) { bh[i] = 0.0; } // Initialize by for (i = 0; i < VOCAB_SIZE; ++i) { by[i] = 0.0; } } void clip(double *array, int len) { for (int i = 0; i < len; ++i) { array[i] = array[i] > CLIP_VALUE ? CLIP_VALUE : array[i] < -CLIP_VALUE ? -CLIP_VALUE : array[i]; } } double softmax(double *x, int size, int i) { int j; double sum_exp = 0.0; for (j = 0; j < size; ++j) { sum_exp += exp(x[j]); } return exp(x[i]) / sum_exp; } void one_hot_encode(char character, double *encoded) { int index = character; for (int i = 0; i < VOCAB_SIZE; ++i) { encoded[i] = (i == index) ? 1.0 : 0.0; } } char one_hot_decode(double *encoded) { int index = 0; double max_val = encoded[0]; for (int i = 1; i < VOCAB_SIZE; ++i) { if (encoded[i] > max_val) { index = i; max_val = encoded[i]; } } return (char)index; } void forward_pass(char *input_sequence, double *hprev, double *loss) { double xs[SEQ_LENGTH][VOCAB_SIZE]; // Input sequence double hs[SEQ_LENGTH + 1][HIDDEN_SIZE]; // Hidden states double ys[SEQ_LENGTH][VOCAB_SIZE]; // Output probabilities double ps[SEQ_LENGTH][VOCAB_SIZE]; // Output values after softmax // Initialize hidden state for (int i = 0; i < HIDDEN_SIZE; ++i) { hs[0][i] = hprev[i]; } // Forward pass for each character in the input sequence for (int t = 0; t < SEQ_LENGTH; ++t) { one_hot_encode(input_sequence[t], xs[t]); // Hidden state calculation for (int i = 0; i < HIDDEN_SIZE; ++i) { double sum = 0.0; for (int j = 0; j < VOCAB_SIZE; ++j) { sum += Wxh[i][j] * xs[t][j]; } sum += bh[i]; hs[t + 1][i] = tanh(sum); } // Output calculation for (int i = 0; i < VOCAB_SIZE; ++i) { double sum = 0.0; for (int j = 0; j < HIDDEN_SIZE; ++j) { sum += Why[i][j] * hs[t + 1][j]; } sum += by[i]; ys[t][i] = sum; ps[t][i] = softmax(ys[t], VOCAB_SIZE, i); } } // Calculate loss *loss = 0.0; for (int t = 0; t < SEQ_LENGTH; ++t) { *loss += -log(ps[t][input_sequence[t]]); } }This is the continuation of the implementation of a CharRNN in C. The
forward_passfunction is added to perform the forward pass of the recurrent neural network.The forward pass starts by initializing the hidden state using the previous hidden state
hprev. Then, for each character in the input sequence, we perform the following steps:- Encode the character using the
one_hot_encodefunction and store it inxs[t]. - Calculate the hidden state at time step
tby multiplying the input character with the corresponding weightsWxhand adding the biasbh. Apply thetanhactivation function to get the hidden state valuehs[t + 1]. - Calculate the output probabilities by multiplying the hidden state
hs[t + 1]with the corresponding weightsWhyand adding the biasby. Store the output probabilities inys[t]. - Apply the softmax function to the output probabilities
ys[t]to get the output valuesps[t].
After the forward pass, the function calculates the loss by summing the negative logarithm of the predicted output values
ps[t][input_sequence[t]]for each time stept.Note that the
hprevparameter is assumed to be the hidden state from the previous time step. This allows you to continue generating text by passing the output hidden state of a previous sequence as input to the next sequence. - Encode the character using the
- User
Now implement the backward pass function.
- Assistant
void backward_pass(char *input_sequence, double *hprev, double *dhnext, double *dWxh, double *dWhh, double *dWhy, double *dbh, double *dby) { int t; // Initialize gradients double dy[VOCAB_SIZE]; double dh[HIDDEN_SIZE]; double dhraw[HIDDEN_SIZE]; double dhs[HIDDEN_SIZE][HIDDEN_SIZE]; double dxs[SEQ_LENGTH][VOCAB_SIZE]; // Initialize gradients to zero for (int i = 0; i < VOCAB_SIZE; ++i) { dy[i] = 0.0; if (i < HIDDEN_SIZE) { dh[i] = 0.0; dhraw[i] = 0.0; dbh[i] = 0.0; } if (i < VOCAB_SIZE) { dby[i] = 0.0; } for (int j = 0; j < VOCAB_SIZE; ++j) { if (i < HIDDEN_SIZE) { dWxh[i][j] = 0.0; if (j < HIDDEN_SIZE) { dWhh[i][j] = 0.0; } } if (j < HIDDEN_SIZE) { dWhy[i][j] = 0.0; } } } // Backward pass for each character in the input sequence for (t = SEQ_LENGTH - 1; t >= 0; --t) { // Calculate gradients of output probabilities for (int i = 0; i < VOCAB_SIZE; ++i) { if (i == input_sequence[t]) { dy[i] = -1.0 + softmax(ys[t], VOCAB_SIZE, i); } else { dy[i] = softmax(ys[t], VOCAB_SIZE, i); } } // Backpropagate through the output layer for (int i = 0; i < VOCAB_SIZE; ++i) { for (int j = 0; j < HIDDEN_SIZE; ++j) { dWhy[i][j] += dy[i] * hs[t + 1][j]; } dby[i] += dy[i]; } // Calculate gradients of hidden states for (int i = 0; i < HIDDEN_SIZE; ++i) { for (int j = 0; j < VOCAB_SIZE; ++j) { dh[i] += dy[j] * Why[j][i]; } } // Backpropagate through the hidden layer for (int i = 0; i < HIDDEN_SIZE; ++i) { dhraw[i] = (1.0 - hs[t + 1][i] * hs[t + 1][i]) * dh[i]; dbh[i] += dhraw[i]; } // Calculate gradients of weights and biases for (int i = 0; i < HIDDEN_SIZE; ++i) { for (int j = 0; j < VOCAB_SIZE; ++j) { dWxh[i][j] += dhraw[i] * xs[t][j]; } } for (int i = 0; i < HIDDEN_SIZE; ++i) { for (int j = 0; j < HIDDEN_SIZE; ++j) { dWhh[i][j] += dhraw[i] * hs[t][j]; } } } // Clip gradients to avoid exploding gradients clip(dWxh, HIDDEN_SIZE * VOCAB_SIZE); clip(dWhh, HIDDEN_SIZE * HIDDEN_SIZE); clip(dWhy, VOCAB_SIZE * HIDDEN_SIZE); clip(dbh, HIDDEN_SIZE); clip(dby, VOCAB_SIZE); // Provide the final hidden state gradient for the next sequence for (int i = 0; i < HIDDEN_SIZE; ++i) { dhnext[i] = dh[i]; } }This is the implementation of the backward pass function. It takes the input sequence, previous hidden state
hprev, and previous hidden state gradientdhnextas inputs, and computes the gradients with respect to the model's parameters and biases.The function starts by initializing the gradients for all parameters and biases to zero. Then, for each character in the input sequence (starting from the last character and moving backwards), the following steps are performed:
- Calculate the gradients of the output probabilities
dyby subtracting 1.0 from the softmax value of the correct output (if it matches the current input character) and taking the softmax of all other outputs. - Compute the gradients of the output weights
dWhyand biasesdbyby multiplyingdywith the corresponding hidden state values from the forward pass. - Backpropagate the gradients through the hidden layer by calculating the gradients of the hidden states
dhusing the output gradientsdyand the weightsWhy. - Compute the gradients of the hidden layer weights
dWxh,dWhh, and biasesdbhby multiplying the gradients of the hidden statesdhwith the corresponding input character valuesxsor hidden state valueshsfrom the forward pass. - Clip the gradients to avoid exploding gradients using the
clipfunction. - Store the final hidden state gradient
dhnextfor the next sequence.
Note that the gradients
dWxh,dWhh,dWhy,dbh, anddbyare passed as pointers to the function, allowing them to be updated directly in the caller function. - Calculate the gradients of the output probabilities