package main import ( "bufio" "encoding/csv" "encoding/gob" "fmt" "log" "os"…
- User
package main
import ( "bufio" "encoding/csv" "encoding/gob" "fmt" "log" "os" "path/filepath" "regexp" "sort" "strings"
"gorgonia.org/gorgonia" "gorgonia.org/tensor")
type ParsedData struct { Input string Output string }
const maxTrainMessages = 10 const maxVocabSize = 500000
func LoadCSVFiles(dirname string) ([]ParsedData, error) { var data []ParsedData fmt.Println("Loading data from directory:", dirname)
files, err := os.ReadDir(dirname) if err != nil { return nil, fmt.Errorf("error reading directory: %v", err) } for _, file := range files { if strings.HasSuffix(file.Name(), ".csv") { filepath := filepath.Join(dirname, file.Name()) fmt.Println("Reading file:", filepath) f, err := os.Open(filepath) if err != nil { return nil, fmt.Errorf("error opening file %s: %v", filepath, err) } r := csv.NewReader(f) records, err := r.ReadAll() f.Close() // closing the file after reading if err != nil { return nil, fmt.Errorf("error reading CSV data from file %s: %v", filepath, err) } for i := 1; i < len(records); i++ { record := records[i] if len(record) >= 6 { data = append(data, ParsedData{ Input: record[4], Output: record[5], }) } } } } fmt.Printf("Loaded %d records from %s\n", len(data), dirname) return data, nil}
func Tokenize(text string) []string { reg, err := regexp.Compile("[^a-zA-Z0-9А-Яа-я]+") if err != nil { log.Fatalf("Regex compilation failed: %v", err) } cleanedText := reg.ReplaceAllString(text, " ") return strings.Fields(strings.ToLower(cleanedText)) }
func BuildVocab(data []ParsedData) map[string]int { vocab := make(map[string]int) wordFreq := make(map[string]int)
for _, item := range data { tokens := append(Tokenize(item.Input), Tokenize(item.Output)...) for _, token := range tokens { wordFreq[token]++ } } type kv struct { Key string Value int } var freqs []kv for k, v := range wordFreq { freqs = append(freqs, kv{k, v}) } sort.Slice(freqs, func(i, j int) bool { return freqs[i].Value > freqs[j].Value }) if len(freqs) > maxVocabSize { freqs = freqs[:maxVocabSize] } for i, kv := range freqs { vocab[kv.Key] = i } return vocab}
func Vectorize(tokens []string, vocab map[string]int) []float32 { vector := make([]float32, len(vocab)) for _, token := range tokens { if idx, exists := vocab[token]; exists { vector[idx] += 1 } } return vector }
func SaveWeights(weights *tensor.Dense, filename string) error { f, err := os.Create(filename) if err != nil { return err } defer f.Close()
encoder := gob.NewEncoder(f) return encoder.Encode(weights)}
func LoadWeights(filename string) (*tensor.Dense, error) { f, err := os.Open(filename) if err != nil { return nil, err } defer f.Close()
var weights tensor.Dense decoder := gob.NewDecoder(f) if err := decoder.Decode(&weights); err != nil { return nil, err } return &weights, nil}
func SaveVocabulary(vocab map[string]int, filename string) error { f, err := os.Create(filename) if err != nil { return err } defer f.Close()
encoder := gob.NewEncoder(f) return encoder.Encode(vocab)}
func LoadVocabulary(filename string) (map[string]int, error) { f, err := os.Open(filename) if err != nil { return nil, err } defer f.Close()
var vocab map[string]int decoder := gob.NewDecoder(f) if err := decoder.Decode(&vocab); err != nil { return nil, err } return vocab, nil}
func trainModel(trainDir string, maxTrainMessages int) { parsedData, err := LoadCSVFiles(trainDir) if err != nil { log.Fatalf("Error loading CSV files: %v", err) }
if maxTrainMessages > len(parsedData) { maxTrainMessages = len(parsedData) } parsedData = parsedData[:maxTrainMessages] fmt.Printf("Using %d training records for modeling\n", len(parsedData)) vocab := BuildVocab(parsedData) if err := SaveVocabulary(vocab, "vocab.gob"); err != nil { log.Fatalf("Error saving vocabulary: %v", err) } batchSize := 150 totalSamples := len(parsedData) numEpochs := 10 g := gorgonia.NewGraph() inputSize := len(vocab) w := gorgonia.NewMatrix(g, tensor.Float32, gorgonia.WithShape(inputSize, 1), gorgonia.WithInit(gorgonia.GlorotN(1.0))) optimizer := gorgonia.NewAdamSolver(gorgonia.WithLearnRate(0.001)) machine := gorgonia.NewTapeMachine(g, gorgonia.BindDualValues(w)) defer machine.Close() for epoch := 0; epoch < numEpochs; epoch++ { for start := 0; start < totalSamples; start += batchSize { end := start + batchSize if end > totalSamples { end = totalSamples } batchSizeActual := end - start xVal := tensor.New(tensor.WithShape(batchSizeActual, inputSize), tensor.Of(tensor.Float32)) yVal := tensor.New(tensor.WithShape(batchSizeActual, 1), tensor.Of(tensor.Float32)) for i := start; i < end; i++ { tokens := Tokenize(parsedData[i].Input) inputVec := Vectorize(tokens, vocab) offset := (i - start) * inputSize copy(xVal.Data().([]float32)[offset:], inputVec) outputValue := float32(0) if parsedData[i].Output != "" { outputValue = float32(parsedData[i].Output[0]) } yVal.SetAt(outputValue, i-start, 0) } x := gorgonia.NewMatrix(g, tensor.Float32, gorgonia.WithShape(batchSizeActual, inputSize), gorgonia.WithValue(xVal)) y := gorgonia.NewMatrix(g, tensor.Float32, gorgonia.WithShape(batchSizeActual, 1), gorgonia.WithValue(yVal)) yhat := gorgonia.Must(gorgonia.Mul(x, w)) loss := gorgonia.Must(gorgonia.Mean(gorgonia.Must(gorgonia.Square(gorgonia.Must(gorgonia.Sub(yhat, y)))))) if _, err := gorgonia.Grad(loss, w); err != nil { log.Fatalf("Failed to create computation graph: %v", err) } if err := machine.RunAll(); err != nil { log.Fatalf("Machine run error: %v", err) } optimizer.Step(gorgonia.NodesToValueGrads([]*gorgonia.Node{w})) machine.Reset() totalBatches := (totalSamples + batchSize - 1) / batchSize totalProgress := float64(epoch*totalBatches+start/batchSize+1) / float64(totalBatches*numEpochs) * 100 progress := float64(end) / float64(totalSamples) * 100 fmt.Printf("Epoch %d: Processed batch from %d to %d (%.2f%% Epoch complete)(%.2f%% complete overall)\n", epoch+1, start, end, progress, totalProgress) } } fmt.Println("Model training complete") if err := SaveWeights(w.Value().(*tensor.Dense), "model_weights.gob"); err != nil { log.Fatalf("Error saving model weights: %v", err) } else { fmt.Println("Model weights saved successfully.") }}
func interactWithModel() { weightsTensor, err := LoadWeights("model_weights.gob") if err != nil { log.Fatalf("Error loading model weights: %v", err) }
vocab, err := LoadVocabulary("vocab.gob") if err != nil { log.Fatalf("Error loading vocabulary: %v", err) } reverseVocab := make(map[int]string) for word, idx := range vocab { reverseVocab[idx] = word } inputSize := len(vocab) reader := bufio.NewReader(os.Stdin) g := gorgonia.NewGraph() w := gorgonia.NewMatrix(g, tensor.Float32, gorgonia.WithShape(inputSize, 1), gorgonia.WithValue(weightsTensor)) for { fmt.Print("Ask a question (or type 'exit' to quit): ") text, _ := reader.ReadString('\n') text = strings.TrimSpace(text) if text == "exit" { break } tokens := Tokenize(text) inputVec := Vectorize(tokens, vocab) xVal := tensor.New(tensor.WithShape(1, inputSize), tensor.Of(tensor.Float32)) copy(xVal.Data().([]float32), inputVec) x := gorgonia.NewMatrix(g, tensor.Float32, gorgonia.WithShape(1, inputSize), gorgonia.WithValue(xVal)) yhat := gorgonia.Must(gorgonia.Mul(x, w)) machine := gorgonia.NewTapeMachine(g) defer machine.Close() if err := machine.RunAll(); err != nil { log.Fatalf("Machine run error: %v", err) } predictionsVal := yhat.Value().Data().([]float32) topIndices := getTopIndices(predictionsVal, 5) var responseTokens []string for _, idx := range topIndices { if word, exists := reverseVocab[idx]; exists { responseTokens = append(responseTokens, word) } } response := strings.Join(responseTokens, " ") fmt.Printf("Generated Response: %s\n", response) machine.Reset() }}
func validateModel(validationDir string) { weightsTensor, err := LoadWeights("model_weights.gob") if err != nil { log.Fatalf("Error loading model weights: %v", err) }
vocab, err := LoadVocabulary("vocab.gob") if err != nil { log.Fatalf("Error loading vocabulary: %v", err) } validationData, err := LoadCSVFiles(validationDir) if err != nil { log.Fatalf("Error loading validation files: %v", err) } g := gorgonia.NewGraph() w := gorgonia.NewMatrix(g, tensor.Float32, gorgonia.WithShape(len(vocab), 1), gorgonia.WithValue(weightsTensor)) correct := 0 total := len(validationData) for _, item := range validationData { inputTokens := Tokenize(item.Input) inputVec := Vectorize(inputTokens, vocab) xVal := tensor.New(tensor.WithShape(1, len(vocab)), tensor.Of(tensor.Float32)) copy(xVal.Data().([]float32), inputVec) x := gorgonia.NewMatrix(g, tensor.Float32, gorgonia.WithShape(1, len(vocab)), gorgonia.WithValue(xVal)) yhat := gorgonia.Must(gorgonia.Mul(x, w)) machine := gorgonia.NewTapeMachine(g) defer machine.Close() if err := machine.RunAll(); err != nil { log.Fatalf("Machine run error: %v", err) } predictedOutput := yhat.Value().Data().([]float32)[0] expectedOutput := float32(0) if item.Output != "" { expectedOutput = float32(item.Output[0]) } if (predictedOutput >= 0.5 && expectedOutput == 1) || (predictedOutput < 0.5 && expectedOutput == 0) { correct++ } machine.Reset() } accuracy := float64(correct) / float64(total) * 100 fmt.Printf("Validation Accuracy: %.2f%%\n", accuracy)}
func getTopIndices(values []float32, n int) []int { type kv struct { Key int Value float32 }
var ss []kv for i, val := range values { ss = append(ss, kv{i, val}) } sort.Slice(ss, func(i, j int) bool { return ss[i].Value > ss[j].Value }) var indices []int for i := 0; i < n && i < len(ss); i++ { indices = append(indices, ss[i].Key) } return indices}
func main() { fmt.Print("Enter mode of operation (t for training, v for validation, d for dialogue): ") scanner := bufio.NewScanner(os.Stdin) scanner.Scan() mode := scanner.Text()
switch mode { case "t": trainModel("train", maxTrainMessages) case "v": validateModel("valid") case "d": interactWithModel() default: fmt.Println("Unknown mode. Use 't' for training, 'v' for validation, or 'd' for dialogue.") }}
исправь валидацмию она должна проходить на базе обучений в фалйах а сейчас сколько бы я не выбирал maxTrainMessages не 10 не 100000000 в валидации всегда 30.11% при v будут также браться файлы csv или txt и по сохраненным данным обучения будет валидироваться то есть проверятьс на сколько ИИ может отвечать на вопросы
- Assistant
To address the issue with validation accuracy, you need to ensure that the validation logic is correctly processing the validation data using the learned model weights and vocabulary. Let's walk through potential issues and solutions for improving the validation phase.
-
Ensure Correct Validation Data Loading:
- Verify that the
validationDirpoints to the correct directory with appropriate CSV or TXT files. - Make sure
LoadCSVFilesis properly capturing the input-output pairs for validation.
- Verify that the
-
Check Tokenization and Vectorization:
- Confirm that the tokenization and vectorization steps in the validation phase are consistent with those used in the training phase. Any discrepancy might lead to poor performance.
-
Assessment Approach:
- Review how you determine correctness in your validation. The current logic seems to convert string outputs to float and expects them to be comparable, which might not align well depending on your labels (e.g., binary classification).
- Consider using other methods to evaluate outputs, such as comparing strings or using classification thresholds appropriate to your problem domain.
-
Revisit Model Architecture:
- Even though you're using a simple linear model, if the expected output is categorical or more complex, you might need different predictions handling or even a more complex model.
-
Debug Logging:
- To gain insights into issues, add debugging print statements to assert that inputs and outputs are as expected at different stages.
Here's a revised sketch with comments to address points mentioned:
func validateModel(validationDir string) { weightsTensor, err := LoadWeights("model_weights.gob") if err != nil { log.Fatalf("Error loading model weights: %v", err) } vocab, err := LoadVocabulary("vocab.gob") if err != nil { log.Fatalf("Error loading vocabulary: %v", err) } validationData, err := LoadCSVFiles(validationDir) if err != nil { log.Fatalf("Error loading validation files: %v", err) } g := gorgonia.NewGraph() w := gorgonia.NewMatrix(g, tensor.Float32, gorgonia.WithShape(len(vocab), 1), gorgonia.WithValue(weightsTensor)) correct := 0 total := len(validationData) for _, item := range validationData { // Ensure tokenization and vectorization are consistent with training inputTokens := Tokenize(item.Input) inputVec := Vectorize(inputTokens, vocab) xVal := tensor.New(tensor.WithShape(1, len(vocab)), tensor.Of(tensor.Float32)) copy(xVal.Data().([]float32), inputVec) x := gorgonia.NewMatrix(g, tensor.Float32, gorgonia.WithShape(1, len(vocab)), gorgonia.WithValue(xVal)) yhat := gorgonia.Must(gorgonia.Mul(x, w)) machine := gorgonia.NewTapeMachine(g) if err := machine.RunAll(); err != nil { log.Fatalf("Machine run error: %v", err) } defer machine.Close() // Evaluate model output predictedOutput := yhat.Value().Data().([]float32)[0] // Interpret expected output // Ensure this comparison matches how outputs have been learned and evaluated expectedOutput := float32(0) if item.Output != "" { expectedOutput = interpretOutput(item.Output) } // Comparison logic might depend on binary classification or other objectives if (predictedOutput >= 0.5 && expectedOutput >= 0.5) || (predictedOutput < 0.5 && expectedOutput < 0.5) { correct++ } machine.Reset() } accuracy := float64(correct) / float64(total) * 100 fmt.Printf("Validation Accuracy: %.2f%%\n", accuracy) } // interpretOutput should convert the CSV output into a comparable float32 value func interpretOutput(output string) float32 { // Customize this logic based on how you define output expectations return float32(output[0]) }Ensure that the interpretOutput function correctly maps your validation data's output to the model's expected output format.
Final Steps
- Confirm training and validation datasets are correctly populated.
- Perform extensive logging at each stage to ensure data flows as intended.
- After adjustments, rerun with varying sizes of training data to observe effects on validation metrics.
-
- User
в valid также может быть и txt файлы и таке может быть много папок всяких в которых тодже етсь csv или txt файлы и данные из valid одже надо токенизоровать
- Assistant
To handle various file types (like CSV and TXT) within potentially nested directories in the
validdirectory, you'll need to extend your function to traverse directories recursively and appropriately handle each file type. Here's how you can adapt the validation function to meet these requirements:-
Recursive Directory Traversal: Use a recursive approach to traverse through directories to find CSV and TXT files.
-
Uniform Data Loading: Create a function that can handle both CSV and TXT files, extracting necessary data for validation.
-
Tokenization and Vectorization: Ensure tokenization and vectorization are consistently applied regardless of file type.
Here's a revised approach that includes handling TXT files and traversing directories:
package main import ( "bufio" "encoding/csv" "fmt" "log" "os" "path/filepath" "regexp" "strings" "gorgonia.org/gorgonia" "gorgonia.org/tensor" ) type ParsedData struct { Input string Output string } func LoadFilesRecursively(dirname string) ([]ParsedData, error) { var data []ParsedData err := filepath.Walk(dirname, func(path string, info os.FileInfo, err error) error { if err != nil { return err } if !info.IsDir() { switch filepath.Ext(path) { case ".csv": fileData, err := loadCSVFile(path) if err != nil { return err } data = append(data, fileData...) case ".txt": fileData, err := loadTXTFile(path) if err != nil { return err } data = append(data, fileData...) } } return nil }) if err != nil { return nil, err } return data, nil } func loadCSVFile(filepath string) ([]ParsedData, error) { var data []ParsedData f, err := os.Open(filepath) if err != nil { return nil, err } defer f.Close() reader := csv.NewReader(f) records, err := reader.ReadAll() if err != nil { return nil, err } for i := 1; i < len(records); i++ { record := records[i] if len(record) >= 2 { // Ensure there are at least two fields for input and output data = append(data, ParsedData{ Input: record[0], Output: record[1], }) } } return data, nil } func loadTXTFile(filepath string) ([]ParsedData, error) { var data []ParsedData f, err := os.Open(filepath) if err != nil { return nil, err } defer f.Close() scanner := bufio.NewScanner(f) for scanner.Scan() { line := scanner.Text() parts := strings.SplitN(line, "\t", 2) // Assuming tab-separated input-output pairs if len(parts) == 2 { data = append(data, ParsedData{ Input: parts[0], Output: parts[1], }) } } if err := scanner.Err(); err != nil { return nil, err } return data, nil } func tokenize(text string) []string { reg, err := regexp.Compile("[^a-zA-Z0-9А-Яа-я]+") if err != nil { log.Fatalf("Regex compilation failed: %v", err) } cleanedText := reg.ReplaceAllString(text, " ") return strings.Fields(strings.ToLower(cleanedText)) } func vectorize(tokens []string, vocab map[string]int) []float32 { vector := make([]float32, len(vocab)) for _, token := range tokens { if idx, exists := vocab[token]; exists { vector[idx] += 1 } } return vector } func validateModel(validationDir string) { weightsTensor, err := LoadWeights("model_weights.gob") if err != nil { log.Fatalf("Error loading model weights: %v", err) } vocab, err := LoadVocabulary("vocab.gob") if err != nil { log.Fatalf("Error loading vocabulary: %v", err) } validationData, err := LoadFilesRecursively(validationDir) if err != nil { log.Fatalf("Error loading validation data: %v", err) } g := gorgonia.NewGraph() w := gorgonia.NewMatrix(g, tensor.Float32, gorgonia.WithShape(len(vocab), 1), gorgonia.WithValue(weightsTensor)) correct := 0 total := len(validationData) for _, item := range validationData { inputTokens := tokenize(item.Input) inputVec := vectorize(inputTokens, vocab) xVal := tensor.New(tensor.WithShape(1, len(vocab)), tensor.Of(tensor.Float32)) copy(xVal.Data().([]float32), inputVec) x := gorgonia.NewMatrix(g, tensor.Float32, gorgonia.WithShape(1, len(vocab)), gorgonia.WithValue(xVal)) yhat := gorgonia.Must(gorgonia.Mul(x, w)) machine := gorgonia.NewTapeMachine(g) if err := machine.RunAll(); err != nil { log.Fatalf("Machine run error: %v", err) } defer machine.Close() predictedOutput := yhat.Value().Data().([]float32)[0] expectedOutput := interpretOutput(item.Output) if predictedOutput == expectedOutput { correct++ } machine.Reset() } accuracy := float64(correct) / float64(total) * 100 fmt.Printf("Validation Accuracy: %.2f%%\n", accuracy) } func interpretOutput(output string) float32 { // Placeholder: interpret output as needed or expected. Modify the logic here based on your target labels. return float32(output[0]) } func main() { fmt.Print("Enter mode of operation (t for training, v for validation, d for dialogue): ") scanner := bufio.NewScanner(os.Stdin) scanner.Scan() mode := scanner.Text() switch mode { case "t": trainModel("train", maxTrainMessages) case "v": validateModel("valid") case "d": interactWithModel() default: fmt.Println("Unknown mode. Use 't' for training, 'v' for validation, or 'd' for dialogue.") } }Key Changes:
- Recursive File Loading: The
LoadFilesRecursivelyfunction usesfilepath.Walkto traverse directories and load all eligible files. - File Handlers: Specific functions
loadCSVFileandloadTXTFilehandle each file type. - Flexible Validation Processing: Ensures that every file's data is tokenized and vectorized before running through the model for validation.
Make sure to adapt
interpretOutputto match the logic of how your model training represented and expected output data. The placeholder logic compares direct values which might not be directly applicable based on your actual dataset setup. -
- User
нет напиши скрипть целиком и полностью
- Assistant
Certainly! Below is a full Go script that incorporates recursive directory traversal for loading CSV and TXT files, applies consistent tokenization and vectorization, and evaluates a model trained with Gorgonia against validation data.
package main import ( "bufio" "encoding/csv" "encoding/gob" "fmt" "log" "os" "path/filepath" "regexp" "strings" "gorgonia.org/gorgonia" "gorgonia.org/tensor" ) type ParsedData struct { Input string Output string } func LoadFilesRecursively(dirname string) ([]ParsedData, error) { var data []ParsedData err := filepath.Walk(dirname, func(path string, info os.FileInfo, err error) error { if err != nil { return err } if !info.IsDir() { switch filepath.Ext(path) { case ".csv": fileData, err := loadCSVFile(path) if err != nil { return err } data = append(data, fileData...) case ".txt": fileData, err := loadTXTFile(path) if err != nil { return err } data = append(data, fileData...) } } return nil }) if err != nil { return nil, err } return data, nil } func loadCSVFile(filepath string) ([]ParsedData, error) { var data []ParsedData f, err := os.Open(filepath) if err != nil { return nil, err } defer f.Close() reader := csv.NewReader(f) records, err := reader.ReadAll() if err != nil { return nil, err } for i := 0; i < len(records); i++ { record := records[i] if len(record) >= 2 { data = append(data, ParsedData{ Input: record[0], Output: record[1], }) } } return data, nil } func loadTXTFile(filepath string) ([]ParsedData, error) { var data []ParsedData f, err := os.Open(filepath) if err != nil { return nil, err } defer f.Close() scanner := bufio.NewScanner(f) for scanner.Scan() { line := scanner.Text() parts := strings.SplitN(line, "\t", 2) if len(parts) == 2 { data = append(data, ParsedData{ Input: parts[0], Output: parts[1], }) } } if err := scanner.Err(); err != nil { return nil, err } return data, nil } func Tokenize(text string) []string { reg, err := regexp.Compile("[^a-zA-Z0-9А-Яа-я]+") if err != nil { log.Fatalf("Regex compilation failed: %v", err) } cleanedText := reg.ReplaceAllString(text, " ") return strings.Fields(strings.ToLower(cleanedText)) } func Vectorize(tokens []string, vocab map[string]int) []float32 { vector := make([]float32, len(vocab)) for _, token := range tokens { if idx, exists := vocab[token]; exists { vector[idx] += 1 } } return vector } func LoadWeights(filename string) (*tensor.Dense, error) { f, err := os.Open(filename) if err != nil { return nil, err } defer f.Close() var weights tensor.Dense decoder := gob.NewDecoder(f) if err := decoder.Decode(&weights); err != nil { return nil, err } return &weights, nil } func LoadVocabulary(filename string) (map[string]int, error) { f, err := os.Open(filename) if err != nil { return nil, err } defer f.Close() var vocab map[string]int decoder := gob.NewDecoder(f) if err := decoder.Decode(&vocab); err != nil { return nil, err } return vocab, nil } func validateModel(validationDir string) { weightsTensor, err := LoadWeights("model_weights.gob") if err != nil { log.Fatalf("Error loading model weights: %v", err) } vocab, err := LoadVocabulary("vocab.gob") if err != nil { log.Fatalf("Error loading vocabulary: %v", err) } validationData, err := LoadFilesRecursively(validationDir) if err != nil { log.Fatalf("Error loading validation data: %v", err) } g := gorgonia.NewGraph() w := gorgonia.NewMatrix(g, tensor.Float32, gorgonia.WithShape(len(vocab), 1), gorgonia.WithValue(weightsTensor)) correct := 0 total := len(validationData) for _, item := range validationData { inputTokens := Tokenize(item.Input) inputVec := Vectorize(inputTokens, vocab) xVal := tensor.New(tensor.WithShape(1, len(vocab)), tensor.Of(tensor.Float32)) copy(xVal.Data().([]float32), inputVec) x := gorgonia.NewMatrix(g, tensor.Float32, gorgonia.WithShape(1, len(vocab)), gorgonia.WithValue(xVal)) yhat := gorgonia.Must(gorgonia.Mul(x, w)) machine := gorgonia.NewTapeMachine(g) if err := machine.RunAll(); err != nil { log.Fatalf("Machine run error: %v", err) } defer machine.Close() predictedOutput := yhat.Value().Data().([]float32)[0] expectedOutput := interpretOutput(item.Output) if predictedOutput == expectedOutput { correct++ } machine.Reset() } accuracy := float64(correct) / float64(total) * 100 fmt.Printf("Validation Accuracy: %.2f%%\n", accuracy) } func interpretOutput(output string) float32 { // Customize this logic based on your label encoding return float32(output[0]) } func main() { fmt.Print("Enter mode of operation (t for training, v for validation, d for dialogue): ") scanner := bufio.NewScanner(os.Stdin) scanner.Scan() mode := scanner.Text() switch mode { case "t": trainModel("train", maxTrainMessages) case "v": validateModel("valid") case "d": interactWithModel() default: fmt.Println("Unknown mode. Use 't' for training, 'v' for validation, or 'd' for dialogue.") } }Key Components Explained:
- Recursive File Loading: Uses
filepath.Walkto find all CSV and TXT files within the specified directory and subdirectories. - File Handlers: Implements separate logic to load data from CSV (
loadCSVFile) and TXT (loadTXTFile) files. - Tokenization and Vectorization: Ensures text data is consistently preprocessed into tokens and vectors using the trained vocabulary.
- Model Validation: Loads model weights and vocabulary, then calculates accuracy over validation data.
Customization Needed:
interpretOutputFunction: Adjust how the expected output is interpreted and compared to predictions to suit your label format.- Training Module: Ensure the
trainModelfunction and vocabulary/weights saving logic are implemented similar to the earlier code for completeness.
Note:
- The
interpretOutputfunction should map output labels from text to whatever format the model expects, altering its logic based on your specific dataset. - This script assumes a simplistic numeric interpretation of labels; if your outputs are more complex, you'll need additional parsing and processing.
- Recursive File Loading: Uses
- User
верни train и interactWithModel
- Assistant
Certainly! I'll provide you with the complete script, including the
trainModelandinteractWithModelfunctions, along with handling the recursive loading of CSV and TXT files. This version of the script ensures that both training and interaction with the model can be performed effectively.package main import ( "bufio" "encoding/csv" "encoding/gob" "fmt" "log" "os" "path/filepath" "regexp" "sort" "strings" "gorgonia.org/gorgonia" "gorgonia.org/tensor" ) type ParsedData struct { Input string Output string } const maxTrainMessages = 10 const maxVocabSize = 500000 func LoadFilesRecursively(dirname string) ([]ParsedData, error) { var data []ParsedData err := filepath.Walk(dirname, func(path string, info os.FileInfo, err error) error { if err != nil { return err } if !info.IsDir() { switch filepath.Ext(path) { case ".csv": fileData, err := loadCSVFile(path) if err != nil { return err } data = append(data, fileData...) case ".txt": fileData, err := loadTXTFile(path) if err != nil { return err } data = append(data, fileData...) } } return nil }) if err != nil { return nil, err } return data, nil } func loadCSVFile(filepath string) ([]ParsedData, error) { var data []ParsedData f, err := os.Open(filepath) if err != nil { return nil, err } defer f.Close() reader := csv.NewReader(f) records, err := reader.ReadAll() if err != nil { return nil, err } for i := 0; i < len(records); i++ { record := records[i] if len(record) >= 2 { data = append(data, ParsedData{ Input: record[0], Output: record[1], }) } } return data, nil } func loadTXTFile(filepath string) ([]ParsedData, error) { var data []ParsedData f, err := os.Open(filepath) if err != nil { return nil, err } defer f.Close() scanner := bufio.NewScanner(f) for scanner.Scan() { line := scanner.Text() parts := strings.SplitN(line, "\t", 2) if len(parts) == 2 { data = append(data, ParsedData{ Input: parts[0], Output: parts[1], }) } } if err := scanner.Err(); err != nil { return nil, err } return data, nil } func Tokenize(text string) []string { reg, err := regexp.Compile("[^a-zA-Z0-9А-Яа-я]+") if err != nil { log.Fatalf("Regex compilation failed: %v", err) } cleanedText := reg.ReplaceAllString(text, " ") return strings.Fields(strings.ToLower(cleanedText)) } func BuildVocab(data []ParsedData) map[string]int { vocab := make(map[string]int) wordFreq := make(map[string]int) for _, item := range data { tokens := append(Tokenize(item.Input), Tokenize(item.Output)...) for _, token := range tokens { wordFreq[token]++ } } type kv struct { Key string Value int } var freqs []kv for k, v := range wordFreq { freqs = append(freqs, kv{k, v}) } sort.Slice(freqs, func(i, j int) bool { return freqs[i].Value > freqs[j].Value }) if len(freqs) > maxVocabSize { freqs = freqs[:maxVocabSize] } for i, kv := range freqs { vocab[kv.Key] = i } return vocab } func Vectorize(tokens []string, vocab map[string]int) []float32 { vector := make([]float32, len(vocab)) for _, token := range tokens { if idx, exists := vocab[token]; exists { vector[idx] += 1 } } return vector } func SaveWeights(weights *tensor.Dense, filename string) error { f, err := os.Create(filename) if err != nil { return err } defer f.Close() encoder := gob.NewEncoder(f) return encoder.Encode(weights) } func LoadWeights(filename string) (*tensor.Dense, error) { f, err := os.Open(filename) if err != nil { return nil, err } defer f.Close() var weights tensor.Dense decoder := gob.NewDecoder(f) if err := decoder.Decode(&weights); err != nil { return nil, err } return &weights, nil } func SaveVocabulary(vocab map[string]int, filename string) error { f, err := os.Create(filename) if err != nil { return err } defer f.Close() encoder := gob.NewEncoder(f) return encoder.Encode(vocab) } func LoadVocabulary(filename string) (map[string]int, error) { f, err := os.Open(filename) if err != nil { return nil, err } defer f.Close() var vocab map[string]int decoder := gob.NewDecoder(f) if err := decoder.Decode(&vocab); err != nil { return nil, err } return vocab, nil } func trainModel(trainDir string, maxTrainMessages int) { parsedData, err := LoadFilesRecursively(trainDir) if err != nil { log.Fatalf("Error loading training data: %v", err) } if maxTrainMessages > len(parsedData) { maxTrainMessages = len(parsedData) } parsedData = parsedData[:maxTrainMessages] fmt.Printf("Using %d training records for modeling\n", len(parsedData)) vocab := BuildVocab(parsedData) if err := SaveVocabulary(vocab, "vocab.gob"); err != nil { log.Fatalf("Error saving vocabulary: %v", err) } batchSize := 150 totalSamples := len(parsedData) numEpochs := 10 g := gorgonia.NewGraph() inputSize := len(vocab) w := gorgonia.NewMatrix(g, tensor.Float32, gorgonia.WithShape(inputSize, 1), gorgonia.WithInit(gorgonia.GlorotN(1.0))) optimizer := gorgonia.NewAdamSolver(gorgonia.WithLearnRate(0.001)) machine := gorgonia.NewTapeMachine(g, gorgonia.BindDualValues(w)) defer machine.Close() for epoch := 0; epoch < numEpochs; epoch++ { for start := 0; start < totalSamples; start += batchSize { end := start + batchSize if end > totalSamples { end = totalSamples } batchSizeActual := end - start xVal := tensor.New(tensor.WithShape(batchSizeActual, inputSize), tensor.Of(tensor.Float32)) yVal := tensor.New(tensor.WithShape(batchSizeActual, 1), tensor.Of(tensor.Float32)) for i := start; i < end; i++ { tokens := Tokenize(parsedData[i].Input) inputVec := Vectorize(tokens, vocab) offset := (i - start) * inputSize copy(xVal.Data().([]float32)[offset:], inputVec) outputValue := float32(0) if parsedData[i].Output != "" { outputValue = float32(parsedData[i].Output[0]) } yVal.SetAt(outputValue, i-start, 0) } x := gorgonia.NewMatrix(g, tensor.Float32, gorgonia.WithShape(batchSizeActual, inputSize), gorgonia.WithValue(xVal)) y := gorgonia.NewMatrix(g, tensor.Float32, gorgonia.WithShape(batchSizeActual, 1), gorgonia.WithValue(yVal)) yhat := gorgonia.Must(gorgonia.Mul(x, w)) loss := gorgonia.Must(gorgonia.Mean(gorgonia.Must(gorgonia.Square(gorgonia.Must(gorgonia.Sub(yhat, y)))))) if _, err := gorgonia.Grad(loss, w); err != nil { log.Fatalf("Failed to create computation graph: %v", err) } if err := machine.RunAll(); err != nil { log.Fatalf("Machine run error: %v", err) } optimizer.Step(gorgonia.NodesToValueGrads([]*gorgonia.Node{w})) machine.Reset() totalBatches := (totalSamples + batchSize - 1) / batchSize totalProgress := float64(epoch*totalBatches+start/batchSize+1) / float64(totalBatches*numEpochs) * 100 progress := float64(end) / float64(totalSamples) * 100 fmt.Printf("Epoch %d: Processed batch from %d to %d (%.2f%% Epoch complete)(%.2f%% complete overall)\n", epoch+1, start, end, progress, totalProgress) } } fmt.Println("Model training complete") if err := SaveWeights(w.Value().(*tensor.Dense), "model_weights.gob"); err != nil { log.Fatalf("Error saving model weights: %v", err) } else { fmt.Println("Model weights saved successfully.") } } func interactWithModel() { weightsTensor, err := LoadWeights("model_weights.gob") if err != nil { log.Fatalf("Error loading model weights: %v", err) } vocab, err := LoadVocabulary("vocab.gob") if err != nil { log.Fatalf("Error loading vocabulary: %v", err) } reverseVocab := make(map[int]string) for word, idx := range vocab { reverseVocab[idx] = word } inputSize := len(vocab) reader := bufio.NewReader(os.Stdin) g := gorgonia.NewGraph() w := gorgonia.NewMatrix(g, tensor.Float32, gorgonia.WithShape(inputSize, 1), gorgonia.WithValue(weightsTensor)) for { fmt.Print("Ask a question (or type 'exit' to quit): ") text, _ := reader.ReadString('\n') text = strings.TrimSpace(text) if text == "exit" { break } tokens := Tokenize(text) inputVec := Vectorize(tokens, vocab) xVal := tensor.New(tensor.WithShape(1, inputSize), tensor.Of(tensor.Float32)) copy(xVal.Data().([]float32), inputVec) x := gorgonia.NewMatrix(g, tensor.Float32, gorgonia.WithShape(1, inputSize), gorgonia.WithValue(xVal)) yhat := gorgonia.Must(gorgonia.Mul(x, w)) machine := gorgonia.NewTapeMachine(g) defer machine.Close() if err := machine.RunAll(); err != nil { log.Fatalf("Machine run error: %v", err) } predictionsVal := yhat.Value().Data().([]float32) topIndices := getTopIndices(predictionsVal, 5) var responseTokens []string for _, idx := range topIndices { if word, exists := reverseVocab[idx]; exists { responseTokens = append(responseTokens, word) } } response := strings.Join(responseTokens, " ") fmt.Printf("Generated Response: %s\n", response) machine.Reset() } } func getTopIndices(values []float32, n int) []int { type kv struct { Key int Value float32 } var ss []kv for i, val := range values { ss = append(ss, kv{i, val}) } sort.Slice(ss, func(i, j int) bool { return ss[i].Value > ss[j].Value }) var indices []int for i := 0; i < n && i < len(ss); i++ { indices = append(indices, ss[i].Key) } return indices } func main() { fmt.Print("Enter mode of operation (t for training, v for validation, d for dialogue): ") scanner := bufio.NewScanner(os.Stdin) scanner.Scan() mode := scanner.Text() switch mode { case "t": trainModel("train", maxTrainMessages) case "v": validateModel("valid") case "d": interactWithModel() default: fmt.Println("Unknown mode. Use 't' for training, 'v' for validation, or 'd' for dialogue.") } }Explanation:
- Training Mode (
trainModel): It reads data from specified directory, builds a vocabulary, vectorizes the data, trains a linear model, and saves the trained model's weights. - Dialogue Mode (
interactWithModel): Reads user input, vectorizes it using the vocabulary, and generates a response by passing it through the trained model. - Recursive Directory Traversal: Handles loading of CSV and TXT files from potentially nested directories for training and validation data.
- Model Validation and Interaction: Uses pre-trained weights and vocabulary for validation and interaction.
Note: Adjust
interpretOutputand input/output handling according to your dataset and model's expected data format. This script assumes basic numeric handling of outputs for simplicity, which might need to be expanded for more complex datasets. - Training Mode (