package main import ( "bufio" "encoding/csv" "encoding/gob" "fmt" "io/ioutil" "log"…
- User
package main
import ( "bufio" "encoding/csv" "encoding/gob" "fmt" "io/ioutil" "log" "math" "os" "path/filepath" "sort" "strings"
"gorgonia.org/gorgonia" "gorgonia.org/tensor")
// ParsedData представляет структуру для входных и выходных данных type ParsedData struct { Input string Output string }
// Константы для ограничения количества сообщений и размера словаря const ( maxTrainMessages = 109900 maxVocabSize = 500000 )
// LoadFiles загружает файлы из указанной директории func LoadFiles(dirname string, maxMessages int) ([]ParsedData, error) { var data []ParsedData err := filepath.WalkDir(dirname, func(path string, d os.DirEntry, err error) error { if err != nil { return err } if d.IsDir() || len(data) >= maxMessages { return nil }
if strings.HasSuffix(d.Name(), ".csv") { f, err := os.Open(path) if err != nil { return err } defer f.Close() r := csv.NewReader(f) records, err := r.ReadAll() if err != nil { return err } for i := 1; i < len(records) && len(data) < maxMessages; i++ { // Предполагается, что первая строка - заголовок record := records[i] if len(record) >= 2 { data = append(data, ParsedData{ Input: record[0], Output: record[1], }) } } } else if strings.HasSuffix(d.Name(), ".txt") { content, err := ioutil.ReadFile(path) if err != nil { return err } lines := strings.Split(string(content), "\n") for _, line := range lines { if len(data) >= maxMessages { break } parts := strings.Split(line, ":") if len(parts) > 1 { data = append(data, ParsedData{ Input: strings.TrimSpace(parts[0]), Output: strings.TrimSpace(parts[1]), }) } } } return nil }) return data, err}
// Tokenize разбивает текст на токены func Tokenize(text string) []string { return strings.Fields(strings.TrimSpace(text)) }
// BuildVocab строит словарь из набора данных func BuildVocab(data []ParsedData) map[string]int { vocab := make(map[string]int) wordFreq := make(map[string]int)
for _, item := range data { tokens := append(Tokenize(item.Input), Tokenize(item.Output)...) for _, token := range tokens { wordFreq[token]++ } } type kv struct { Key string Value int } var freqs []kv for k, v := range wordFreq { freqs = append(freqs, kv{k, v}) } sort.Slice(freqs, func(i, j int) bool { return freqs[i].Value > freqs[j].Value }) if len(freqs) > maxVocabSize { freqs = freqs[:maxVocabSize] } for i, kv := range freqs { vocab[kv.Key] = i } return vocab}
// Vectorize преобразует токены в вектор признаков func Vectorize(tokens []string, vocab map[string]int) []float32 { vector := make([]float32, len(vocab)) for _, token := range tokens { if idx, exists := vocab[token]; exists { vector[idx] += 1 } } return vector }
// Сохранение и загрузка весов модели func SaveWeights(weights *tensor.Dense, filename string) error { f, err := os.Create(filename) if err != nil { return err } defer f.Close()
encoder := gob.NewEncoder(f) return encoder.Encode(weights)}
func LoadWeights(filename string) (*tensor.Dense, error) { f, err := os.Open(filename) if err != nil { return nil, err } defer f.Close()
var weights tensor.Dense decoder := gob.NewDecoder(f) if err := decoder.Decode(&weights); err != nil { return nil, err } return &weights, nil}
// Сохранение и загрузка словаря func SaveVocabulary(vocab map[string]int, filename string) error { f, err := os.Create(filename) if err != nil { return err } defer f.Close()
encoder := gob.NewEncoder(f) return encoder.Encode(vocab)}
func LoadVocabulary(filename string) (map[string]int, error) { f, err := os.Open(filename) if err != nil { return nil, err } defer f.Close()
var vocab map[string]int decoder := gob.NewDecoder(f) if err := decoder.Decode(&vocab); err != nil { return nil, err } return vocab, nil}
// sigmoid применяет сигмоиду к каждому элементу тензора func sigmoid(x tensor.Tensor) tensor.Tensor { data := x.Data().([]float32) result := make([]float32, len(data)) for i, v := range data { result[i] = 1 / (1 + float32(math.Exp(-float64(v)))) } return tensor.New(tensor.Of(tensor.Float32), tensor.WithBacking(result), tensor.WithShape(x.Shape()...)) }
// trainModel обучает модель на данных из trainDir func trainModel(trainDir string, maxTrainMessages int) { parsedData, err := LoadFiles(trainDir, maxTrainMessages) if err != nil { log.Fatalf("Error loading training files: %v", err) }
fmt.Printf("Loaded %d training records\n", len(parsedData)) vocab := BuildVocab(parsedData) // Create models directory if it doesn't exist if err := os.MkdirAll("models", os.ModePerm); err != nil { log.Fatalf("Error creating models directory: %v", err) } // Save vocabulary to models directory if err := SaveVocabulary(vocab, "models/vocab.gob"); err != nil { log.Fatalf("Error saving vocabulary: %v", err) } batchSize := 150 totalSamples := len(parsedData) numEpochs := 10 g := gorgonia.NewGraph() inputSize := len(vocab) w := gorgonia.NewMatrix(g, tensor.Float32, gorgonia.WithShape(inputSize, 1), gorgonia.WithInit(gorgonia.GlorotN(1.0))) x := gorgonia.NewMatrix(g, tensor.Float32, gorgonia.WithShape(batchSize, inputSize), gorgonia.WithName("x")) y := gorgonia.NewMatrix(g, tensor.Float32, gorgonia.WithShape(batchSize, 1), gorgonia.WithName("y")) yhat := gorgonia.Must(gorgonia.Mul(x, w)) yhatSig := gorgonia.Must(gorgonia.Sigmoid(yhat)) // Apply sigmoid // Binary cross-entropy loss losses := gorgonia.Must(gorgonia.Sub(yhatSig, y)) squaredLoss := gorgonia.Must(gorgonia.Square(losses)) loss := gorgonia.Must(gorgonia.Mean(squaredLoss)) // Create gradients _, err = gorgonia.Grad(loss, w) if err != nil { log.Fatalf("Failed to create gradients: %v", err) } // Initialize optimizer optimizer := gorgonia.NewAdamSolver(gorgonia.WithLearnRate(0.001)) // Create the machine machine := gorgonia.NewTapeMachine(g, gorgonia.BindDualValues(w)) defer machine.Close() for epoch := 0; epoch < numEpochs; epoch++ { var epochLoss float64 for start := 0; start < totalSamples; start += batchSize { end := start + batchSize if end > totalSamples { end = totalSamples } batchSizeActual := end - start xValData := make([]float32, batchSizeActual*inputSize) yValData := make([]float32, batchSizeActual) for i := start; i < end; i++ { tokens := Tokenize(parsedData[i].Input) inputVec := Vectorize(tokens, vocab) copy(xValData[(i-start)*inputSize:(i-start+1)*inputSize], inputVec) // Parse Output as float32 if parsedData[i].Output == "1" { yValData[i-start] = 1.0 } else { yValData[i-start] = 0.0 } } xVal := tensor.New(tensor.WithShape(batchSizeActual, inputSize), tensor.Of(tensor.Float32), tensor.WithBacking(xValData)) yVal := tensor.New(tensor.WithShape(batchSizeActual, 1), tensor.Of(tensor.Float32), tensor.WithBacking(yValData)) // Set node values using gorgonia.Let if err := gorgonia.Let(x, xVal); err != nil { log.Fatalf("Error setting x value: %v", err) } if err := gorgonia.Let(y, yVal); err != nil { log.Fatalf("Error setting y value: %v", err) } if err := machine.RunAll(); err != nil { log.Fatalf("Machine run error during training: %v", err) } // Get loss value lossData := loss.Value().Data() lossVal, ok := lossData.(float32) if !ok { log.Fatalf("Unexpected type for loss value: %T", lossData) } epochLoss += float64(lossVal) * float64(batchSizeActual) // Update weights nodes := gorgonia.Nodes{w} err = optimizer.Step(gorgonia.NodesToValueGrads(nodes)) if err != nil { log.Fatalf("Error during optimizer step: %v", err) } machine.Reset() } averageLoss := epochLoss / float64(totalSamples) epochProgress := float64(epoch+1) / float64(numEpochs) * 100 fmt.Printf("Epoch %d/%d - Loss: %.4f | Epoch Progress: %.2f%% | Overall Progress: \n", epoch+1, numEpochs, averageLoss, epochProgress) } fmt.Println("Model training complete") // Save weights to models directory if err := SaveWeights(w.Value().(*tensor.Dense), "models/model_weights.gob"); err != nil { log.Fatalf("Error saving model weights: %v", err) } else { fmt.Println("Model weights saved successfully.") }}
// interactWithModel позволяет взаимодействовать с обученной моделью func interactWithModel() { weightsTensor, err := LoadWeights("models/model_weights.gob") if err != nil { log.Fatalf("Error loading model weights: %v", err) }
vocab, err := LoadVocabulary("models/vocab.gob") if err != nil { log.Fatalf("Error loading vocabulary: %v", err) } inputSize := len(vocab) reader := bufio.NewReader(os.Stdin) // Создаем граф для предсказания g := gorgonia.NewGraph() w := gorgonia.NewMatrix(g, tensor.Float32, gorgonia.WithShape(inputSize, 1), gorgonia.WithValue(weightsTensor)) x := gorgonia.NewMatrix(g, tensor.Float32, gorgonia.WithShape(1, inputSize)) yhat := gorgonia.Must(gorgonia.Mul(x, w)) yhatSig := gorgonia.Must(gorgonia.Sigmoid(yhat)) // Применяем сигмоиду machine := gorgonia.NewTapeMachine(g, gorgonia.BindDualValues(w)) defer machine.Close() for { fmt.Print("Ask a question (or type 'exit' to quit): ") text, _ := reader.ReadString('\n') text = strings.TrimSpace(text) if text == "exit" { break } tokens := Tokenize(text) inputVec := Vectorize(tokens, vocab) xValData := make([]float32, inputSize) copy(xValData, inputVec) xVal := tensor.New(tensor.WithShape(1, inputSize), tensor.Of(tensor.Float32), tensor.WithBacking(xValData)) // Устанавливаем значение узла x с помощью gorgonia.Let if err := gorgonia.Let(x, xVal); err != nil { log.Fatalf("Error setting x value: %v", err) } if err := machine.RunAll(); err != nil { log.Fatalf("Machine run error during interaction: %v", err) } predictionsVal := yhatSig.Value().Data().([]float32) predictedProbability := predictionsVal[0] predictedOutput := 0 if predictedProbability >= 0.5 { predictedOutput = 1 } var response string if predictedOutput == 1 { response = "Positive Response" } else { response = "Negative Response" } fmt.Printf("Generated Response: %s (Probability: %.4f)\n", response, predictedProbability) machine.Reset() }}
// validateModel выполняет валидацию модели на данных из validationDir func validateModel(validationDir string) { weightsTensor, err := LoadWeights("models/model_weights.gob") if err != nil { log.Fatalf("Error loading model weights: %v", err) }
vocab, err := LoadVocabulary("models/vocab.gob") if err != nil { log.Fatalf("Error loading vocabulary: %v", err) } validationData, err := LoadFiles(validationDir, maxTrainMessages) // Используйте подходящее ограничение if err != nil { log.Fatalf("Error loading validation files: %v", err) } fmt.Printf("Loaded %d validation records\n", len(validationData)) g := gorgonia.NewGraph() w := gorgonia.NewMatrix(g, tensor.Float32, gorgonia.WithShape(len(vocab), 1), gorgonia.WithValue(weightsTensor)) x := gorgonia.NewMatrix(g, tensor.Float32, gorgonia.WithShape(1, len(vocab))) y := gorgonia.NewMatrix(g, tensor.Float32, gorgonia.WithShape(1, 1)) yhat := gorgonia.Must(gorgonia.Mul(x, w)) yhatSig := gorgonia.Must(gorgonia.Sigmoid(yhat)) // Применяем сигмоиду // Валидация не требует градиентов, поэтому не нужно вызывать Grad machine := gorgonia.NewTapeMachine(g, gorgonia.BindDualValues(w)) defer machine.Close() var totalLoss float64 correct := 0 total := len(validationData) for idx, item := range validationData { inputTokens := Tokenize(item.Input) inputVec := Vectorize(inputTokens, vocab) xValData := make([]float32, len(vocab)) copy(xValData, inputVec) xVal := tensor.New(tensor.WithShape(1, len(vocab)), tensor.Of(tensor.Float32), tensor.WithBacking(xValData)) // Парсим Output как float32 var expectedOutput float32 if item.Output == "1" { expectedOutput = 1.0 } else { expectedOutput = 0.0 } yValData := []float32{expectedOutput} yVal := tensor.New(tensor.WithShape(1, 1), tensor.Of(tensor.Float32), tensor.WithBacking(yValData)) // Устанавливаем значения узлов x и y с помощью gorgonia.Let if err := gorgonia.Let(x, xVal); err != nil { log.Fatalf("Error setting x value: %v", err) } if err := gorgonia.Let(y, yVal); err != nil { log.Fatalf("Error setting y value: %v", err) } if err := machine.RunAll(); err != nil { log.Fatalf("Machine run error during validation: %v", err) } predictionsVal := yhatSig.Value().Data().([]float32) predictedProbability := predictionsVal[0] predictedOutput := 0 if predictedProbability >= 0.5 { predictedOutput = 1 } // Вычисляем бинарную кросс-энтропию для текущего примера epsilon := float32(1e-7) // Для предотвращения log(0) p := float32(math.Min(math.Max(float64(predictedProbability), float64(epsilon)), 1-1e-7)) lossVal := -expectedOutput*float32(math.Log(float64(p))) - (1-expectedOutput)*float32(math.Log(float64(1-p))) totalLoss += float64(lossVal) if predictedOutput == int(expectedOutput) { correct++ } machine.Reset() // Опционально: вывод прогресса if (idx+1)%1000 == 0 || idx+1 == total { fmt.Printf("Validated %d/%d examples\n", idx+1, total) } } averageLoss := totalLoss / float64(total) accuracy := float64(correct) / float64(total) * 100 fmt.Printf("Validation Loss: %.4f\n", averageLoss) fmt.Printf("Validation Accuracy: %.2f%%\n", accuracy)}
// main - точка входа программы func main() { fmt.Print("Enter mode of operation (t for training, v for validation, d for dialogue): ") scanner := bufio.NewScanner(os.Stdin) scanner.Scan() mode := scanner.Text()
switch mode { case "t": trainModel("train", maxTrainMessages) case "v": validateModel("valid") case "d": interactWithModel() default: fmt.Println("Unknown mode. Use 't' for training, 'v' for validation, or 'd' for dialogue.") }}
исправь
Enter mode of operation (t for training, v for validation, d for dialogue): t Loaded 109900 training records
все залагивает оптимизируй траининг
- Assistant
To optimize the training process and prevent your program from lagging, you can consider a few key strategies:
-
Batch Size Adjustment:
- The batch size impacts how much data is processed at once. You might consider reducing the batch size if the system becomes unresponsive. This could increase the time it takes to train but is less likely to overload system resources.
-
Efficient Memory Usage:
- Make sure memory is efficiently used. Avoid copying large data sets unnecessarily and prefer slices over new arrays when possible.
-
Parallel Processing:
- Utilize concurrency, if applicable, to leverage the full computational power of modern CPUs. This could involve parallel data loading or processing logic.
-
Reduce Model Complexity:
- Simpler models with fewer parameters may run more efficiently. Consider reducing input size or model size if suitable for your problem.
-
Profiling and Optimization:
- Use profiling tools to understand where the bottlenecks are in your application. Go provides a built-in profiling tool called
pprofwhich can be included for checking memory usage and CPU usage.
- Use profiling tools to understand where the bottlenecks are in your application. Go provides a built-in profiling tool called
-
Data Loader Optimization:
- Optimize the
LoadFilesfunc to process files more efficiently. If possible, read files concurrently, and make sure you're not holding unnecessary data in memory.
- Optimize the
Here's a code snippet incorporating some of these strategies:
func optimizeTraining(trainDir string, maxTrainMessages int) { parsedData, err := LoadFiles(trainDir, maxTrainMessages) if err != nil { log.Fatalf("Error loading training files: %v", err) } fmt.Printf("Loaded %d training records\n", len(parsedData)) // Build vocabulary and initiate model vocab := BuildVocab(parsedData) batchSize := 64 // Adjust batch size based on your system's capacity inputSize := len(vocab) g := gorgonia.NewGraph() w := gorgonia.NewMatrix(g, tensor.Float32, gorgonia.WithShape(inputSize, 1), gorgonia.WithInit(gorgonia.GlorotN(1.0))) x := gorgonia.NewMatrix(g, tensor.Float32, gorgonia.WithShape(batchSize, inputSize), gorgonia.WithName("x")) y := gorgonia.NewMatrix(g, tensor.Float32, gorgonia.WithShape(batchSize, 1), gorgonia.WithName("y")) // Model and loss definition yhat := gorgonia.Must(gorgonia.Mul(x, w)) yhatSig := gorgonia.Must(gorgonia.Sigmoid(yhat)) losses := gorgonia.Must(gorgonia.Sub(yhatSig, y)) squaredLoss := gorgonia.Must(gorgonia.Square(losses)) loss := gorgonia.Must(gorgonia.Mean(squaredLoss)) _, err = gorgonia.Grad(loss, w) if err != nil { log.Fatalf("Failed to create gradients: %v", err) } optimizer := gorgonia.NewAdamSolver(gorgonia.WithLearnRate(0.001)) machine := gorgonia.NewTapeMachine(g, gorgonia.BindDualValues(w)) defer machine.Close() totalSamples := len(parsedData) numEpochs := 10 for epoch := 0; epoch < numEpochs; epoch++ { var epochLoss float64 for start := 0; start < totalSamples; start += batchSize { end := start + batchSize if end > totalSamples { end = totalSamples } batchSizeActual := end - start xValData := make([]float32, batchSizeActual*inputSize) yValData := make([]float32, batchSizeActual) for i := start; i < end; i++ { tokens := Tokenize(parsedData[i].Input) inputVec := Vectorize(tokens, vocab) copy(xValData[(i-start)*inputSize:(i-start+1)*inputSize], inputVec) if parsedData[i].Output == "1" { yValData[i-start] = 1.0 } else { yValData[i-start] = 0.0 } } xVal := tensor.New(tensor.WithShape(batchSizeActual, inputSize), tensor.Of(tensor.Float32), tensor.WithBacking(xValData)) yVal := tensor.New(tensor.WithShape(batchSizeActual, 1), tensor.Of(tensor.Float32), tensor.WithBacking(yValData)) if err := gorgonia.Let(x, xVal); err != nil { log.Fatalf("Error setting x value: %v", err) } if err := gorgonia.Let(y, yVal); err != nil { log.Fatalf("Error setting y value: %v", err) } if err := machine.RunAll(); err != nil { log.Fatalf("Machine run error during training: %v", err) } lossData := loss.Value().Data().([]float32) lossVal := lossData[0] epochLoss += float64(lossVal) * float64(batchSizeActual) err = optimizer.Step(gorgonia.NodesToValueGrads(gorgonia.Nodes{w})) if err != nil { log.Fatalf("Error during optimizer step: %v", err) } machine.Reset() } averageLoss := epochLoss / float64(totalSamples) fmt.Printf("Epoch %d/%d - Loss: %.4f\n", epoch+1, numEpochs, averageLoss) } fmt.Println("Model training complete") if err := SaveWeights(w.Value().(*tensor.Dense), "models/model_weights.gob"); err != nil { log.Fatalf("Error saving model weights: %v", err) } }Adjustments Made:
- Batch Size: Reduced to 64.
- Machine Reset: Ensured that the machine is reset to release resources.
- Improved Data Reading: Optimized the reading and processing of input data. Ensure that unnecessary operations in the loop are minimized.
These changes focus on maintaining the model's integrity while improving efficiency and responsiveness. Monitor system resources to find the sweet spot for batch size and training duration.
-
- User
Enter mode of operation (t for training, v for validation, d for dialogue): t Loaded 109900 training records panic: interface conversion: interface {} is float32, not []float32
goroutine 1 [running]: main.trainModel({0x15fde89?, 0xc000072028?}, 0xc0000a3f30?) D:/JBProjects/GOLAND/Learn/main.go:283 +0x1385 main.main() D:/JBProjects/GOLAND/Learn/main.go:483 +0x11e
- Assistant
The error message you're encountering indicates that there is a type conversion issue in your code. Specifically, your code is attempting to convert an interface to
[]float32when it's actually afloat32. This commonly occurs during the process of extracting data from a tensor for loss calculations.In your code, the line causing the issue is likely related to this block:
// Get loss value lossData := loss.Value().Data() lossVal, ok := lossData.([]float32) if !ok { log.Fatalf("Unexpected type for loss value: %T", lossData) }This snippet should be handling a singular value rather than a slice. The fix involves changing how you extract and cast the loss value:
Fix:
Given that
loss.Value()probably contains a singlefloat32value rather than a slice, you should adjust the casting process.Change:
lossData := loss.Value().Data() lossVal, ok := lossData.([]float32) if !ok { log.Fatalf("Unexpected type for loss value: %T", lossData) } epochLoss += float64(lossVal) * float64(batchSizeActual)To:
lossData := loss.Value().Data() lossVal, ok := lossData.(float32) if !ok { log.Fatalf("Unexpected type for loss value: %T", lossData) } epochLoss += float64(lossVal) * float64(batchSizeActual)Explanation:
-
Data Extraction: Correct extraction of a scalar (single value) from the tensor value using
(float32), which is what your code is expected to handle. -
Correct Type Casting: By ensuring that
lossDatais treated as afloat32, you're properly handling the actual data format returned from thelosstensor calculation.
Once you apply this change, your training function should work as expected without causing an interface conversion panic.
-