package main import ( "bufio" "encoding/csv" "encoding/gob" "fmt" "io/ioutil" "log"…
- User
package main
import ( "bufio" "encoding/csv" "encoding/gob" "fmt" "io/ioutil" "log" "math" "os" "path/filepath" "sort" "strings"
"gorgonia.org/gorgonia" "gorgonia.org/tensor")
// ParsedData представляет структуру для входных и выходных данных type ParsedData struct { Input string Output string }
// Константы для ограничения количества сообщений и размера словаря const ( maxTrainMessages = 4950 maxVocabSize = 5000000 )
// LoadFiles загружает файлы из указанной директории func LoadFiles(dirname string, maxMessages int) ([]ParsedData, error) { var data []ParsedData err := filepath.WalkDir(dirname, func(path string, d os.DirEntry, err error) error { if err != nil { return err } if d.IsDir() || len(data) >= maxMessages { return nil }
if strings.HasSuffix(d.Name(), ".csv") { f, err := os.Open(path) if err != nil { return err } defer f.Close() r := csv.NewReader(f) records, err := r.ReadAll() if err != nil { return err } for i := 1; i < len(records) && len(data) < maxMessages; i++ { // Предполагается, что первая строка - заголовок record := records[i] if len(record) >= 2 { data = append(data, ParsedData{ Input: record[0], Output: record[1], }) } } } else if strings.HasSuffix(d.Name(), ".txt") { content, err := ioutil.ReadFile(path) if err != nil { return err } lines := strings.Split(string(content), "\n") for _, line := range lines { if len(data) >= maxMessages { break } parts := strings.Split(line, ":") if len(parts) > 1 { data = append(data, ParsedData{ Input: strings.TrimSpace(parts[0]), Output: strings.TrimSpace(parts[1]), }) } } } return nil }) return data, err}
// Tokenize разбивает текст на токены func Tokenize(text string) []string { return strings.Fields(strings.TrimSpace(text)) }
// BuildVocab строит словарь из набора данных func BuildVocab(data []ParsedData) map[string]int { vocab := make(map[string]int) wordFreq := make(map[string]int)
for _, item := range data { tokens := append(Tokenize(item.Input), Tokenize(item.Output)...) for _, token := range tokens { wordFreq[token]++ } } type kv struct { Key string Value int } var freqs []kv for k, v := range wordFreq { freqs = append(freqs, kv{k, v}) } sort.Slice(freqs, func(i, j int) bool { return freqs[i].Value > freqs[j].Value }) if len(freqs) > maxVocabSize { freqs = freqs[:maxVocabSize] } for i, kv := range freqs { vocab[kv.Key] = i } return vocab}
// Vectorize преобразует токены в вектор признаков func Vectorize(tokens []string, vocab map[string]int) []float32 { vector := make([]float32, len(vocab)) for _, token := range tokens { if idx, exists := vocab[token]; exists { vector[idx] += 1 } } return vector }
// Сохранение и загрузка весов модели func SaveWeights(weights *tensor.Dense, filename string) error { f, err := os.Create(filename) if err != nil { return err } defer f.Close()
encoder := gob.NewEncoder(f) return encoder.Encode(weights)}
func LoadWeights(filename string) (*tensor.Dense, error) { f, err := os.Open(filename) if err != nil { return nil, err } defer f.Close()
var weights tensor.Dense decoder := gob.NewDecoder(f) if err := decoder.Decode(&weights); err != nil { return nil, err } return &weights, nil}
// Сохранение и загрузка словаря func SaveVocabulary(vocab map[string]int, filename string) error { f, err := os.Create(filename) if err != nil { return err } defer f.Close()
encoder := gob.NewEncoder(f) return encoder.Encode(vocab)}
func LoadVocabulary(filename string) (map[string]int, error) { f, err := os.Open(filename) if err != nil { return nil, err } defer f.Close()
var vocab map[string]int decoder := gob.NewDecoder(f) if err := decoder.Decode(&vocab); err != nil { return nil, err } return vocab, nil}
// sigmoid применяет сигмоиду к каждому элементу тензора func sigmoid(x tensor.Tensor) tensor.Tensor { data := x.Data().([]float32) result := make([]float32, len(data)) for i, v := range data { result[i] = 1 / (1 + float32(math.Exp(-float64(v)))) } return tensor.New(tensor.Of(tensor.Float32), tensor.WithBacking(result), tensor.WithShape(x.Shape()...)) }
// trainModel обучает модель на данных из trainDir func trainModel(trainDir string, maxTrainMessages int) { parsedData, err := LoadFiles(trainDir, maxTrainMessages) if err != nil { log.Fatalf("Error loading training files: %v", err) }
fmt.Printf("Loaded %d training records\n", len(parsedData)) vocab := BuildVocab(parsedData) // Create models directory if it doesn't exist if err := os.MkdirAll("models", os.ModePerm); err != nil { log.Fatalf("Error creating models directory: %v", err) } // Save vocabulary to models directory if err := SaveVocabulary(vocab, "models/vocab.gob"); err != nil { log.Fatalf("Error saving vocabulary: %v", err) } batchSize := 150 totalSamples := len(parsedData) numEpochs := 10 g := gorgonia.NewGraph() inputSize := len(vocab) w := gorgonia.NewMatrix(g, tensor.Float32, gorgonia.WithShape(inputSize, 1), gorgonia.WithInit(gorgonia.GlorotN(1.0))) x := gorgonia.NewMatrix(g, tensor.Float32, gorgonia.WithShape(batchSize, inputSize), gorgonia.WithName("x")) y := gorgonia.NewMatrix(g, tensor.Float32, gorgonia.WithShape(batchSize, 1), gorgonia.WithName("y")) yhat := gorgonia.Must(gorgonia.Mul(x, w)) yhatSig := gorgonia.Must(gorgonia.Sigmoid(yhat)) // Apply sigmoid // Binary cross-entropy loss losses := gorgonia.Must(gorgonia.Sub(yhatSig, y)) squaredLoss := gorgonia.Must(gorgonia.Square(losses)) loss := gorgonia.Must(gorgonia.Mean(squaredLoss)) // Create gradients _, err = gorgonia.Grad(loss, w) if err != nil { log.Fatalf("Failed to create gradients: %v", err) } // Initialize optimizer optimizer := gorgonia.NewAdamSolver(gorgonia.WithLearnRate(0.001)) // Create the machine machine := gorgonia.NewTapeMachine(g, gorgonia.BindDualValues(w)) defer machine.Close() for epoch := 0; epoch < numEpochs; epoch++ { var epochLoss float64 for start := 0; start < totalSamples; start += batchSize { end := start + batchSize if end > totalSamples { end = totalSamples } batchSizeActual := end - start xValData := make([]float32, batchSizeActual*inputSize) yValData := make([]float32, batchSizeActual) for i := start; i < end; i++ { tokens := Tokenize(parsedData[i].Input) inputVec := Vectorize(tokens, vocab) copy(xValData[(i-start)*inputSize:(i-start+1)*inputSize], inputVec) // Parse Output as float32 if parsedData[i].Output == "1" { yValData[i-start] = 1.0 } else { yValData[i-start] = 0.0 } } xVal := tensor.New(tensor.WithShape(batchSizeActual, inputSize), tensor.Of(tensor.Float32), tensor.WithBacking(xValData)) yVal := tensor.New(tensor.WithShape(batchSizeActual, 1), tensor.Of(tensor.Float32), tensor.WithBacking(yValData)) // Set node values using gorgonia.Let if err := gorgonia.Let(x, xVal); err != nil { log.Fatalf("Error setting x value: %v", err) } if err := gorgonia.Let(y, yVal); err != nil { log.Fatalf("Error setting y value: %v", err) } if err := machine.RunAll(); err != nil { log.Fatalf("Machine run error during training: %v", err) } // Get loss value lossData := loss.Value().Data() lossVal, ok := lossData.(float32) if !ok { log.Fatalf("Unexpected type for loss value: %T", lossData) } epochLoss += float64(lossVal) * float64(batchSizeActual) // Update weights nodes := gorgonia.Nodes{w} err = optimizer.Step(gorgonia.NodesToValueGrads(nodes)) if err != nil { log.Fatalf("Error during optimizer step: %v", err) } machine.Reset() } averageLoss := epochLoss / float64(totalSamples) fmt.Printf("Epoch %d/%d - Loss: %.4f\n", epoch+1, numEpochs, averageLoss) } fmt.Println("Model training complete") // Save weights to models directory if err := SaveWeights(w.Value().(*tensor.Dense), "models/model_weights.gob"); err != nil { log.Fatalf("Error saving model weights: %v", err) } else { fmt.Println("Model weights saved successfully.") }}
// interactWithModel позволяет взаимодействовать с обученной моделью func interactWithModel() { weightsTensor, err := LoadWeights("models/model_weights.gob") if err != nil { log.Fatalf("Error loading model weights: %v", err) }
vocab, err := LoadVocabulary("models/vocab.gob") if err != nil { log.Fatalf("Error loading vocabulary: %v", err) } inputSize := len(vocab) reader := bufio.NewReader(os.Stdin) // Создаем граф для предсказания g := gorgonia.NewGraph() w := gorgonia.NewMatrix(g, tensor.Float32, gorgonia.WithShape(inputSize, 1), gorgonia.WithValue(weightsTensor)) x := gorgonia.NewMatrix(g, tensor.Float32, gorgonia.WithShape(1, inputSize)) yhat := gorgonia.Must(gorgonia.Mul(x, w)) yhatSig := gorgonia.Must(gorgonia.Sigmoid(yhat)) // Применяем сигмоиду machine := gorgonia.NewTapeMachine(g, gorgonia.BindDualValues(w)) defer machine.Close() for { fmt.Print("Ask a question (or type 'exit' to quit): ") text, _ := reader.ReadString('\n') text = strings.TrimSpace(text) if text == "exit" { break } tokens := Tokenize(text) inputVec := Vectorize(tokens, vocab) xValData := make([]float32, inputSize) copy(xValData, inputVec) xVal := tensor.New(tensor.WithShape(1, inputSize), tensor.Of(tensor.Float32), tensor.WithBacking(xValData)) // Устанавливаем значение узла x с помощью gorgonia.Let if err := gorgonia.Let(x, xVal); err != nil { log.Fatalf("Error setting x value: %v", err) } if err := machine.RunAll(); err != nil { log.Fatalf("Machine run error during interaction: %v", err) } predictionsVal := yhatSig.Value().Data().([]float32) predictedProbability := predictionsVal[0] predictedOutput := 0 if predictedProbability >= 0.5 { predictedOutput = 1 } var response string if predictedOutput == 1 { response = "Positive Response" } else { response = "Negative Response" } fmt.Printf("Generated Response: %s (Probability: %.4f)\n", response, predictedProbability) machine.Reset() }}
// validateModel выполняет валидацию модели на данных из validationDir func validateModel(validationDir string) { weightsTensor, err := LoadWeights("models/model_weights.gob") if err != nil { log.Fatalf("Error loading model weights: %v", err) }
vocab, err := LoadVocabulary("models/vocab.gob") if err != nil { log.Fatalf("Error loading vocabulary: %v", err) } validationData, err := LoadFiles(validationDir, maxTrainMessages) // Используйте подходящее ограничение if err != nil { log.Fatalf("Error loading validation files: %v", err) } fmt.Printf("Loaded %d validation records\n", len(validationData)) g := gorgonia.NewGraph() w := gorgonia.NewMatrix(g, tensor.Float32, gorgonia.WithShape(len(vocab), 1), gorgonia.WithValue(weightsTensor)) x := gorgonia.NewMatrix(g, tensor.Float32, gorgonia.WithShape(1, len(vocab))) y := gorgonia.NewMatrix(g, tensor.Float32, gorgonia.WithShape(1, 1)) yhat := gorgonia.Must(gorgonia.Mul(x, w)) yhatSig := gorgonia.Must(gorgonia.Sigmoid(yhat)) // Применяем сигмоиду // Валидация не требует градиентов, поэтому не нужно вызывать Grad machine := gorgonia.NewTapeMachine(g, gorgonia.BindDualValues(w)) defer machine.Close() var totalLoss float64 correct := 0 total := len(validationData) for idx, item := range validationData { inputTokens := Tokenize(item.Input) inputVec := Vectorize(inputTokens, vocab) xValData := make([]float32, len(vocab)) copy(xValData, inputVec) xVal := tensor.New(tensor.WithShape(1, len(vocab)), tensor.Of(tensor.Float32), tensor.WithBacking(xValData)) // Парсим Output как float32 var expectedOutput float32 if item.Output == "1" { expectedOutput = 1.0 } else { expectedOutput = 0.0 } yValData := []float32{expectedOutput} yVal := tensor.New(tensor.WithShape(1, 1), tensor.Of(tensor.Float32), tensor.WithBacking(yValData)) // Устанавливаем значения узлов x и y с помощью gorgonia.Let if err := gorgonia.Let(x, xVal); err != nil { log.Fatalf("Error setting x value: %v", err) } if err := gorgonia.Let(y, yVal); err != nil { log.Fatalf("Error setting y value: %v", err) } if err := machine.RunAll(); err != nil { log.Fatalf("Machine run error during validation: %v", err) } predictionsVal := yhatSig.Value().Data().([]float32) predictedProbability := predictionsVal[0] predictedOutput := 0 if predictedProbability >= 0.5 { predictedOutput = 1 } // Вычисляем бинарную кросс-энтропию для текущего примера epsilon := float32(1e-7) // Для предотвращения log(0) p := float32(math.Min(math.Max(float64(predictedProbability), float64(epsilon)), 1-1e-7)) lossVal := -expectedOutput*float32(math.Log(float64(p))) - (1-expectedOutput)*float32(math.Log(float64(1-p))) totalLoss += float64(lossVal) if predictedOutput == int(expectedOutput) { correct++ } machine.Reset() // Опционально: вывод прогресса if (idx+1)%1000 == 0 || idx+1 == total { fmt.Printf("Validated %d/%d examples\n", idx+1, total) } } averageLoss := totalLoss / float64(total) accuracy := float64(correct) / float64(total) * 100 fmt.Printf("Validation Loss: %.4f\n", averageLoss) fmt.Printf("Validation Accuracy: %.2f%%\n", accuracy)}
// main - точка входа программы func main() { fmt.Print("Enter mode of operation (t for training, v for validation, d for dialogue): ") scanner := bufio.NewScanner(os.Stdin) scanner.Scan() mode := scanner.Text()
switch mode { case "t": trainModel("train", maxTrainMessages) case "v": validateModel("valid") case "d": interactWithModel() default: fmt.Println("Unknown mode. Use 't' for training, 'v' for validation, or 'd' for dialogue.") }}
добавь линию процесса обучения эпохи как в пайтоне при скачивании бибилиотек Epoch --------------- 100%(это процент прогресса эпохи) loss - потеря loss должен добавляться после полгого оубучения эпохи
именно сделай так тчобы было столько сообщений сколько и эпох то еть чтобы ыло Epoch --------------- 100%(это процент прогресса эпохи) Average Loss: 0.2378 именно чтоыб оно изменялось и чтобы показывать прогресс эпохи изменяй - на = или на тлстую линию и меняй процент
получаеться просто перепиши train
- Assistant
To enhance the training process with a visual progress bar similar to those found in Python libraries, you can modify the
trainModelfunction to display an epoch-wise progress bar. This will provide real-time feedback on the training progress, showing the percentage of completion and the average loss at the end of each epoch.Below is the updated
trainModelfunction with the added progress bar functionality:// trainModel обучает модель на данных из trainDir func trainModel(trainDir string, maxTrainMessages int) { parsedData, err := LoadFiles(trainDir, maxTrainMessages) if err != nil { log.Fatalf("Error loading training files: %v", err) } fmt.Printf("Loaded %d training records\n", len(parsedData)) vocab := BuildVocab(parsedData) // Create models directory if it doesn't exist if err := os.MkdirAll("models", os.ModePerm); err != nil { log.Fatalf("Error creating models directory: %v", err) } // Save vocabulary to models directory if err := SaveVocabulary(vocab, "models/vocab.gob"); err != nil { log.Fatalf("Error saving vocabulary: %v", err) } batchSize := 150 totalSamples := len(parsedData) numEpochs := 10 g := gorgonia.NewGraph() inputSize := len(vocab) w := gorgonia.NewMatrix(g, tensor.Float32, gorgonia.WithShape(inputSize, 1), gorgonia.WithInit(gorgonia.GlorotN(1.0))) x := gorgonia.NewMatrix(g, tensor.Float32, gorgonia.WithShape(batchSize, inputSize), gorgonia.WithName("x")) y := gorgonia.NewMatrix(g, tensor.Float32, gorgonia.WithShape(batchSize, 1), gorgonia.WithName("y")) yhat := gorgonia.Must(gorgonia.Mul(x, w)) yhatSig := gorgonia.Must(gorgonia.Sigmoid(yhat)) // Apply sigmoid // Binary cross-entropy loss losses := gorgonia.Must(gorgonia.Sub(yhatSig, y)) squaredLoss := gorgonia.Must(gorgonia.Square(losses)) loss := gorgonia.Must(gorgonia.Mean(squaredLoss)) // Create gradients _, err = gorgonia.Grad(loss, w) if err != nil { log.Fatalf("Failed to create gradients: %v", err) } // Initialize optimizer optimizer := gorgonia.NewAdamSolver(gorgonia.WithLearnRate(0.001)) // Create the machine machine := gorgonia.NewTapeMachine(g, gorgonia.BindDualValues(w)) defer machine.Close() for epoch := 0; epoch < numEpochs; epoch++ { var epochLoss float64 numBatches := int(math.Ceil(float64(totalSamples) / float64(batchSize))) barWidth := 50 // Width of the progress bar // Initialize progress bar fmt.Printf("Epoch %d/%d\n[", epoch+1, numEpochs) for i := 0; i < barWidth; i++ { fmt.Print(" ") } fmt.Print("] 0%") fmt.Print("\r[") // Move cursor back to start of the progress bar for start := 0; start < totalSamples; start += batchSize { end := start + batchSize if end > totalSamples { end = totalSamples } batchSizeActual := end - start xValData := make([]float32, batchSizeActual*inputSize) yValData := make([]float32, batchSizeActual) for i := start; i < end; i++ { tokens := Tokenize(parsedData[i].Input) inputVec := Vectorize(tokens, vocab) copy(xValData[(i-start)*inputSize:(i-start+1)*inputSize], inputVec) // Parse Output as float32 if parsedData[i].Output == "1" { yValData[i-start] = 1.0 } else { yValData[i-start] = 0.0 } } xVal := tensor.New(tensor.WithShape(batchSizeActual, inputSize), tensor.Of(tensor.Float32), tensor.WithBacking(xValData)) yVal := tensor.New(tensor.WithShape(batchSizeActual, 1), tensor.Of(tensor.Float32), tensor.WithBacking(yValData)) // Set node values using gorgonia.Let if err := gorgonia.Let(x, xVal); err != nil { log.Fatalf("Error setting x value: %v", err) } if err := gorgonia.Let(y, yVal); err != nil { log.Fatalf("Error setting y value: %v", err) } if err := machine.RunAll(); err != nil { log.Fatalf("Machine run error during training: %v", err) } // Get loss value lossData := loss.Value().Data() lossVal, ok := lossData.(float32) if !ok { log.Fatalf("Unexpected type for loss value: %T", lossData) } epochLoss += float64(lossVal) * float64(batchSizeActual) // Update weights nodes := gorgonia.Nodes{w} err = optimizer.Step(gorgonia.NodesToValueGrads(nodes)) if err != nil { log.Fatalf("Error during optimizer step: %v", err) } // Update progress bar currentBatch := start + batchSize if currentBatch > totalSamples { currentBatch = totalSamples } percent := float64(currentBatch) / float64(totalSamples) filledLength := int(barWidth * percent) // Move cursor to the progress bar start fmt.Printf("\r[EPOCH %d/%d] [", epoch+1, numEpochs) for i := 0; i < filledLength; i++ { fmt.Print("=") } for i := filledLength; i < barWidth; i++ { fmt.Print(" ") } fmt.Printf("] %3.0f%%", percent*100) machine.Reset() } averageLoss := epochLoss / float64(totalSamples) // Move to the next line after progress bar completes fmt.Printf("\r[EPOCH %d/%d] [", epoch+1, numEpochs) for i := 0; i < barWidth; i++ { fmt.Print("=") } fmt.Printf("] 100%% - Average Loss: %.4f\n", averageLoss) } fmt.Println("Model training complete") // Save weights to models directory if err := SaveWeights(w.Value().(*tensor.Dense), "models/model_weights.gob"); err != nil { log.Fatalf("Error saving model weights: %v", err) } else { fmt.Println("Model weights saved successfully.") } }Explanation of the Changes
-
Progress Bar Parameters:
numBatches: Calculates the total number of batches per epoch.barWidth: Sets the width of the progress bar (number of characters).
-
Initializing the Progress Bar:
- Before processing each epoch, the program prints the epoch number and initializes an empty progress bar:
fmt.Printf("Epoch %d/%d\n[", epoch+1, numEpochs) for i := 0; i < barWidth; i++ { fmt.Print(" ") } fmt.Print("] 0%") fmt.Print("\r[") // Move cursor back to the start of the progress bar
- Before processing each epoch, the program prints the epoch number and initializes an empty progress bar:
-
Updating the Progress Bar:
-
After processing each batch, the progress bar is updated to reflect the current progress:
currentBatch := start + batchSize if currentBatch > totalSamples { currentBatch = totalSamples } percent := float64(currentBatch) / float64(totalSamples) filledLength := int(barWidth * percent) // Move cursor to the progress bar start fmt.Printf("\r[EPOCH %d/%d] [", epoch+1, numEpochs) for i := 0; i < filledLength; i++ { fmt.Print("=") } for i := filledLength; i < barWidth; i++ { fmt.Print(" ") } fmt.Printf("] %3.0f%%", percent*100) -
This code calculates the percentage of completion and fills the progress bar accordingly using
=characters.
-
-
Finalizing the Progress Bar:
- After all batches in an epoch are processed, the progress bar is finalized to 100%, and the average loss is displayed:
averageLoss := epochLoss / float64(totalSamples) // Move to the next line after progress bar completes fmt.Printf("\r[EPOCH %d/%d] [", epoch+1, numEpochs) for i := 0; i < barWidth; i++ { fmt.Print("=") } fmt.Printf("] 100%% - Average Loss: %.4f\n", averageLoss)
- After all batches in an epoch are processed, the progress bar is finalized to 100%, and the average loss is displayed:
-
Cursor Management:
- The
\r(carriage return) is used to reset the cursor to the beginning of the line, allowing the program to overwrite the existing progress bar with the updated one.
- The
Sample Output
Here's how the output will look during training:
Loaded 4950 training records Epoch 1/10 [ ] 0%[EPOCH 1/10] [======== ] 20% [EPOCH 1/10] [================== ] 40% [EPOCH 1/10] [============================== ] 60% [EPOCH 1/10] [======================================== ] 80% [EPOCH 1/10] [==================================================] 100% - Average Loss: 0.2378 Epoch 2/10 [ ] 0%[EPOCH 2/10] [====== ] 12% ... Epoch 10/10 [ ] 0%[EPOCH 10/10] [==================================================] 100% - Average Loss: 0.1234 Model training complete Model weights saved successfully.Each epoch will display a progress bar that fills up as batches are processed. Once the epoch is complete, the average loss for that epoch is displayed.
Additional Improvements (Optional)
-
Smooth Progress Bar:
- For a smoother progress bar, especially with many batches, you might consider updating the progress bar after a certain number of batches instead of every single batch to reduce flickering.
-
Using External Libraries:
- For more advanced progress bars, consider using external libraries such as
github.com/schollz/progressbar. This can provide more features and a cleaner implementation.
- For more advanced progress bars, consider using external libraries such as
-
Handling Large Outputs:
- If the number of epochs or batches is very large, ensure that your terminal can handle the rapid updates without performance issues.
Final Thoughts
By incorporating a progress bar into the training loop, you enhance the user experience by providing real-time feedback on the training process. This modification makes it easier to monitor long-running training tasks and assess the model's performance over time.
-