Skip to content
The Chat Archive
gpt-4o-2024-08-06· OpenAI WildChat-4.8M (AI2)

package main import ( "bufio" "encoding/csv" "encoding/gob" "fmt" "log" "os"…

  1. User

    package main

    import ( "bufio" "encoding/csv" "encoding/gob" "fmt" "log" "os" "path/filepath" "regexp" "sort" "strings"

    "gorgonia.org/gorgonia"
    "gorgonia.org/tensor"
    

    )

    type ParsedData struct { Input string Output string }

    const maxTrainMessages = 10 const maxVocabSize = 500000

    func LoadCSVFiles(dirname string) ([]ParsedData, error) { var data []ParsedData fmt.Println("Loading data from directory:", dirname)

    files, err := os.ReadDir(dirname)
    if err != nil {
    	return nil, fmt.Errorf("error reading directory: %v", err)
    }
    
    for _, file := range files {
    	if strings.HasSuffix(file.Name(), ".csv") {
    		filepath := filepath.Join(dirname, file.Name())
    		fmt.Println("Reading file:", filepath)
    		f, err := os.Open(filepath)
    		if err != nil {
    			return nil, fmt.Errorf("error opening file %s: %v", filepath, err)
    		}
    
    		r := csv.NewReader(f)
    		records, err := r.ReadAll()
    		f.Close() // closing the file after reading
    		if err != nil {
    			return nil, fmt.Errorf("error reading CSV data from file %s: %v", filepath, err)
    		}
    
    		for i := 1; i < len(records); i++ {
    			record := records[i]
    			if len(record) >= 6 {
    				data = append(data, ParsedData{
    					Input:  record[4],
    					Output: record[5],
    				})
    			}
    		}
    	}
    }
    
    fmt.Printf("Loaded %d records from %s\n", len(data), dirname)
    return data, nil
    

    }

    func Tokenize(text string) []string { reg, err := regexp.Compile("[^a-zA-Z0-9А-Яа-я]+") if err != nil { log.Fatalf("Regex compilation failed: %v", err) } cleanedText := reg.ReplaceAllString(text, " ") return strings.Fields(strings.ToLower(cleanedText)) }

    func BuildVocab(data []ParsedData) map[string]int { vocab := make(map[string]int) wordFreq := make(map[string]int)

    for _, item := range data {
    	tokens := append(Tokenize(item.Input), Tokenize(item.Output)...)
    	for _, token := range tokens {
    		wordFreq[token]++
    	}
    }
    
    type kv struct {
    	Key   string
    	Value int
    }
    
    var freqs []kv
    for k, v := range wordFreq {
    	freqs = append(freqs, kv{k, v})
    }
    
    sort.Slice(freqs, func(i, j int) bool {
    	return freqs[i].Value > freqs[j].Value
    })
    
    if len(freqs) > maxVocabSize {
    	freqs = freqs[:maxVocabSize]
    }
    
    for i, kv := range freqs {
    	vocab[kv.Key] = i
    }
    return vocab
    

    }

    func Vectorize(tokens []string, vocab map[string]int) []float32 { vector := make([]float32, len(vocab)) for _, token := range tokens { if idx, exists := vocab[token]; exists { vector[idx] += 1 } } return vector }

    func SaveWeights(weights *tensor.Dense, filename string) error { f, err := os.Create(filename) if err != nil { return err } defer f.Close()

    encoder := gob.NewEncoder(f)
    return encoder.Encode(weights)
    

    }

    func LoadWeights(filename string) (*tensor.Dense, error) { f, err := os.Open(filename) if err != nil { return nil, err } defer f.Close()

    var weights tensor.Dense
    decoder := gob.NewDecoder(f)
    if err := decoder.Decode(&weights); err != nil {
    	return nil, err
    }
    return &weights, nil
    

    }

    func SaveVocabulary(vocab map[string]int, filename string) error { f, err := os.Create(filename) if err != nil { return err } defer f.Close()

    encoder := gob.NewEncoder(f)
    return encoder.Encode(vocab)
    

    }

    func LoadVocabulary(filename string) (map[string]int, error) { f, err := os.Open(filename) if err != nil { return nil, err } defer f.Close()

    var vocab map[string]int
    decoder := gob.NewDecoder(f)
    if err := decoder.Decode(&vocab); err != nil {
    	return nil, err
    }
    return vocab, nil
    

    }

    func trainModel(trainDir string, maxTrainMessages int) { parsedData, err := LoadCSVFiles(trainDir) if err != nil { log.Fatalf("Error loading CSV files: %v", err) }

    if maxTrainMessages > len(parsedData) {
    	maxTrainMessages = len(parsedData)
    }
    
    parsedData = parsedData[:maxTrainMessages]
    fmt.Printf("Using %d training records for modeling\n", len(parsedData))
    
    vocab := BuildVocab(parsedData)
    
    if err := SaveVocabulary(vocab, "vocab.gob"); err != nil {
    	log.Fatalf("Error saving vocabulary: %v", err)
    }
    
    batchSize := 150
    totalSamples := len(parsedData)
    numEpochs := 10
    
    g := gorgonia.NewGraph()
    
    inputSize := len(vocab)
    w := gorgonia.NewMatrix(g, tensor.Float32, gorgonia.WithShape(inputSize, 1), gorgonia.WithInit(gorgonia.GlorotN(1.0)))
    
    optimizer := gorgonia.NewAdamSolver(gorgonia.WithLearnRate(0.001))
    
    machine := gorgonia.NewTapeMachine(g, gorgonia.BindDualValues(w))
    defer machine.Close()
    
    for epoch := 0; epoch < numEpochs; epoch++ {
    	for start := 0; start < totalSamples; start += batchSize {
    		end := start + batchSize
    		if end > totalSamples {
    			end = totalSamples
    		}
    
    		batchSizeActual := end - start
    		xVal := tensor.New(tensor.WithShape(batchSizeActual, inputSize), tensor.Of(tensor.Float32))
    		yVal := tensor.New(tensor.WithShape(batchSizeActual, 1), tensor.Of(tensor.Float32))
    
    		for i := start; i < end; i++ {
    			tokens := Tokenize(parsedData[i].Input)
    			inputVec := Vectorize(tokens, vocab)
    			offset := (i - start) * inputSize
    			copy(xVal.Data().([]float32)[offset:], inputVec)
    
    			outputValue := float32(0)
    			if parsedData[i].Output != "" {
    				outputValue = float32(parsedData[i].Output[0])
    			}
    			yVal.SetAt(outputValue, i-start, 0)
    		}
    
    		x := gorgonia.NewMatrix(g, tensor.Float32, gorgonia.WithShape(batchSizeActual, inputSize), gorgonia.WithValue(xVal))
    		y := gorgonia.NewMatrix(g, tensor.Float32, gorgonia.WithShape(batchSizeActual, 1), gorgonia.WithValue(yVal))
    
    		yhat := gorgonia.Must(gorgonia.Mul(x, w))
    		loss := gorgonia.Must(gorgonia.Mean(gorgonia.Must(gorgonia.Square(gorgonia.Must(gorgonia.Sub(yhat, y))))))
    
    		if _, err := gorgonia.Grad(loss, w); err != nil {
    			log.Fatalf("Failed to create computation graph: %v", err)
    		}
    
    		if err := machine.RunAll(); err != nil {
    			log.Fatalf("Machine run error: %v", err)
    		}
    
    		optimizer.Step(gorgonia.NodesToValueGrads([]*gorgonia.Node{w}))
    
    		machine.Reset()
    
    		totalBatches := (totalSamples + batchSize - 1) / batchSize
    		totalProgress := float64(epoch*totalBatches+start/batchSize+1) / float64(totalBatches*numEpochs) * 100
    		progress := float64(end) / float64(totalSamples) * 100
    		fmt.Printf("Epoch %d: Processed batch from %d to %d (%.2f%% Epoch complete)(%.2f%% complete overall)\n", epoch+1, start, end, progress, totalProgress)
    	}
    }
    
    fmt.Println("Model training complete")
    
    if err := SaveWeights(w.Value().(*tensor.Dense), "model_weights.gob"); err != nil {
    	log.Fatalf("Error saving model weights: %v", err)
    } else {
    	fmt.Println("Model weights saved successfully.")
    }
    

    }

    func interactWithModel() { weightsTensor, err := LoadWeights("model_weights.gob") if err != nil { log.Fatalf("Error loading model weights: %v", err) }

    vocab, err := LoadVocabulary("vocab.gob")
    if err != nil {
    	log.Fatalf("Error loading vocabulary: %v", err)
    }
    
    reverseVocab := make(map[int]string)
    for word, idx := range vocab {
    	reverseVocab[idx] = word
    }
    
    inputSize := len(vocab)
    reader := bufio.NewReader(os.Stdin)
    
    g := gorgonia.NewGraph()
    w := gorgonia.NewMatrix(g, tensor.Float32, gorgonia.WithShape(inputSize, 1), gorgonia.WithValue(weightsTensor))
    
    for {
    	fmt.Print("Ask a question (or type 'exit' to quit): ")
    	text, _ := reader.ReadString('\n')
    	text = strings.TrimSpace(text)
    
    	if text == "exit" {
    		break
    	}
    
    	tokens := Tokenize(text)
    	inputVec := Vectorize(tokens, vocab)
    
    	xVal := tensor.New(tensor.WithShape(1, inputSize), tensor.Of(tensor.Float32))
    	copy(xVal.Data().([]float32), inputVec)
    
    	x := gorgonia.NewMatrix(g, tensor.Float32, gorgonia.WithShape(1, inputSize), gorgonia.WithValue(xVal))
    	yhat := gorgonia.Must(gorgonia.Mul(x, w))
    
    	machine := gorgonia.NewTapeMachine(g)
    	defer machine.Close()
    
    	if err := machine.RunAll(); err != nil {
    		log.Fatalf("Machine run error: %v", err)
    	}
    
    	predictionsVal := yhat.Value().Data().([]float32)
    	topIndices := getTopIndices(predictionsVal, 5)
    
    	var responseTokens []string
    	for _, idx := range topIndices {
    		if word, exists := reverseVocab[idx]; exists {
    			responseTokens = append(responseTokens, word)
    		}
    	}
    
    	response := strings.Join(responseTokens, " ")
    	fmt.Printf("Generated Response: %s\n", response)
    	machine.Reset()
    }
    

    }

    func validateModel(validationDir string) { weightsTensor, err := LoadWeights("model_weights.gob") if err != nil { log.Fatalf("Error loading model weights: %v", err) }

    vocab, err := LoadVocabulary("vocab.gob")
    if err != nil {
    	log.Fatalf("Error loading vocabulary: %v", err)
    }
    
    validationData, err := LoadCSVFiles(validationDir)
    if err != nil {
    	log.Fatalf("Error loading validation files: %v", err)
    }
    
    g := gorgonia.NewGraph()
    w := gorgonia.NewMatrix(g, tensor.Float32, gorgonia.WithShape(len(vocab), 1), gorgonia.WithValue(weightsTensor))
    
    correct := 0
    total := len(validationData)
    
    for _, item := range validationData {
    	inputTokens := Tokenize(item.Input)
    	inputVec := Vectorize(inputTokens, vocab)
    
    	xVal := tensor.New(tensor.WithShape(1, len(vocab)), tensor.Of(tensor.Float32))
    	copy(xVal.Data().([]float32), inputVec)
    
    	x := gorgonia.NewMatrix(g, tensor.Float32, gorgonia.WithShape(1, len(vocab)), gorgonia.WithValue(xVal))
    	yhat := gorgonia.Must(gorgonia.Mul(x, w))
    
    	machine := gorgonia.NewTapeMachine(g)
    	defer machine.Close()
    
    	if err := machine.RunAll(); err != nil {
    		log.Fatalf("Machine run error: %v", err)
    	}
    
    	predictedOutput := yhat.Value().Data().([]float32)[0]
    	expectedOutput := float32(0)
    	if item.Output != "" {
    		expectedOutput = float32(item.Output[0])
    	}
    
    	if (predictedOutput >= 0.5 && expectedOutput == 1) || (predictedOutput < 0.5 && expectedOutput == 0) {
    		correct++
    	}
    
    	machine.Reset()
    }
    
    accuracy := float64(correct) / float64(total) * 100
    fmt.Printf("Validation Accuracy: %.2f%%\n", accuracy)
    

    }

    func getTopIndices(values []float32, n int) []int { type kv struct { Key int Value float32 }

    var ss []kv
    for i, val := range values {
    	ss = append(ss, kv{i, val})
    }
    
    sort.Slice(ss, func(i, j int) bool {
    	return ss[i].Value > ss[j].Value
    })
    
    var indices []int
    for i := 0; i < n && i < len(ss); i++ {
    	indices = append(indices, ss[i].Key)
    }
    return indices
    

    }

    func main() { fmt.Print("Enter mode of operation (t for training, v for validation, d for dialogue): ") scanner := bufio.NewScanner(os.Stdin) scanner.Scan() mode := scanner.Text()

    switch mode {
    case "t":
    	trainModel("train", maxTrainMessages)
    case "v":
    	validateModel("valid")
    case "d":
    	interactWithModel()
    default:
    	fmt.Println("Unknown mode. Use 't' for training, 'v' for validation, or 'd' for dialogue.")
    }
    

    }

    исправь валидацмию она должна проходить на базе обучений в фалйах а сейчас сколько бы я не выбирал maxTrainMessages не 10 не 100000000 в валидации всегда 30.11% при v будут также браться файлы csv или txt и по сохраненным данным обучения будет валидироваться то есть проверятьс на сколько ИИ может отвечать на вопросы

  2. Assistant

    To address the issue with validation accuracy, you need to ensure that the validation logic is correctly processing the validation data using the learned model weights and vocabulary. Let's walk through potential issues and solutions for improving the validation phase.

    1. Ensure Correct Validation Data Loading:

      • Verify that the validationDir points to the correct directory with appropriate CSV or TXT files.
      • Make sure LoadCSVFiles is properly capturing the input-output pairs for validation.
    2. Check Tokenization and Vectorization:

      • Confirm that the tokenization and vectorization steps in the validation phase are consistent with those used in the training phase. Any discrepancy might lead to poor performance.
    3. Assessment Approach:

      • Review how you determine correctness in your validation. The current logic seems to convert string outputs to float and expects them to be comparable, which might not align well depending on your labels (e.g., binary classification).
      • Consider using other methods to evaluate outputs, such as comparing strings or using classification thresholds appropriate to your problem domain.
    4. Revisit Model Architecture:

      • Even though you're using a simple linear model, if the expected output is categorical or more complex, you might need different predictions handling or even a more complex model.
    5. Debug Logging:

      • To gain insights into issues, add debugging print statements to assert that inputs and outputs are as expected at different stages.

    Here's a revised sketch with comments to address points mentioned:

    func validateModel(validationDir string) {
    	weightsTensor, err := LoadWeights("model_weights.gob")
    	if err != nil {
    		log.Fatalf("Error loading model weights: %v", err)
    	}
    
    	vocab, err := LoadVocabulary("vocab.gob")
    	if err != nil {
    		log.Fatalf("Error loading vocabulary: %v", err)
    	}
    
    	validationData, err := LoadCSVFiles(validationDir)
    	if err != nil {
    		log.Fatalf("Error loading validation files: %v", err)
    	}
    
    	g := gorgonia.NewGraph()
    	w := gorgonia.NewMatrix(g, tensor.Float32, gorgonia.WithShape(len(vocab), 1), gorgonia.WithValue(weightsTensor))
    
    	correct := 0
    	total := len(validationData)
    
    	for _, item := range validationData {
    		// Ensure tokenization and vectorization are consistent with training
    		inputTokens := Tokenize(item.Input)
    		inputVec := Vectorize(inputTokens, vocab)
    
    		xVal := tensor.New(tensor.WithShape(1, len(vocab)), tensor.Of(tensor.Float32))
    		copy(xVal.Data().([]float32), inputVec)
    
    		x := gorgonia.NewMatrix(g, tensor.Float32, gorgonia.WithShape(1, len(vocab)), gorgonia.WithValue(xVal))
    		yhat := gorgonia.Must(gorgonia.Mul(x, w))
    
    		machine := gorgonia.NewTapeMachine(g)
    		if err := machine.RunAll(); err != nil {
    			log.Fatalf("Machine run error: %v", err)
    		}
    		defer machine.Close()
    
    		// Evaluate model output
    		predictedOutput := yhat.Value().Data().([]float32)[0]
    
    		// Interpret expected output
    		// Ensure this comparison matches how outputs have been learned and evaluated
    		expectedOutput := float32(0)
    		if item.Output != "" {
    			expectedOutput = interpretOutput(item.Output)
    		}
    
    		// Comparison logic might depend on binary classification or other objectives
    		if (predictedOutput >= 0.5 && expectedOutput >= 0.5) || (predictedOutput < 0.5 && expectedOutput < 0.5) {
    			correct++
    		}
    
    		machine.Reset()
    	}
    
    	accuracy := float64(correct) / float64(total) * 100
    	fmt.Printf("Validation Accuracy: %.2f%%\n", accuracy)
    }
    
    // interpretOutput should convert the CSV output into a comparable float32 value
    func interpretOutput(output string) float32 {
    	// Customize this logic based on how you define output expectations
    	return float32(output[0])
    }
    

    Ensure that the interpretOutput function correctly maps your validation data's output to the model's expected output format.

    Final Steps

    • Confirm training and validation datasets are correctly populated.
    • Perform extensive logging at each stage to ensure data flows as intended.
    • After adjustments, rerun with varying sizes of training data to observe effects on validation metrics.
  3. User

    в valid также может быть и txt файлы и таке может быть много папок всяких в которых тодже етсь csv или txt файлы и данные из valid одже надо токенизоровать

  4. Assistant

    To handle various file types (like CSV and TXT) within potentially nested directories in the valid directory, you'll need to extend your function to traverse directories recursively and appropriately handle each file type. Here's how you can adapt the validation function to meet these requirements:

    1. Recursive Directory Traversal: Use a recursive approach to traverse through directories to find CSV and TXT files.

    2. Uniform Data Loading: Create a function that can handle both CSV and TXT files, extracting necessary data for validation.

    3. Tokenization and Vectorization: Ensure tokenization and vectorization are consistently applied regardless of file type.

    Here's a revised approach that includes handling TXT files and traversing directories:

    package main
    
    import (
    	"bufio"
    	"encoding/csv"
    	"fmt"
    	"log"
    	"os"
    	"path/filepath"
    	"regexp"
    	"strings"
    
    	"gorgonia.org/gorgonia"
    	"gorgonia.org/tensor"
    )
    
    type ParsedData struct {
    	Input  string
    	Output string
    }
    
    func LoadFilesRecursively(dirname string) ([]ParsedData, error) {
    	var data []ParsedData
    	err := filepath.Walk(dirname, func(path string, info os.FileInfo, err error) error {
    		if err != nil {
    			return err
    		}
    
    		if !info.IsDir() {
    			switch filepath.Ext(path) {
    			case ".csv":
    				fileData, err := loadCSVFile(path)
    				if err != nil {
    					return err
    				}
    				data = append(data, fileData...)
    			case ".txt":
    				fileData, err := loadTXTFile(path)
    				if err != nil {
    					return err
    				}
    				data = append(data, fileData...)
    			}
    		}
    		return nil
    	})
    	if err != nil {
    		return nil, err
    	}
    	return data, nil
    }
    
    func loadCSVFile(filepath string) ([]ParsedData, error) {
    	var data []ParsedData
    	f, err := os.Open(filepath)
    	if err != nil {
    		return nil, err
    	}
    	defer f.Close()
    
    	reader := csv.NewReader(f)
    	records, err := reader.ReadAll()
    	if err != nil {
    		return nil, err
    	}
    
    	for i := 1; i < len(records); i++ {
    		record := records[i]
    		if len(record) >= 2 { // Ensure there are at least two fields for input and output
    			data = append(data, ParsedData{
    				Input:  record[0],
    				Output: record[1],
    			})
    		}
    	}
    	return data, nil
    }
    
    func loadTXTFile(filepath string) ([]ParsedData, error) {
    	var data []ParsedData
    	f, err := os.Open(filepath)
    	if err != nil {
    		return nil, err
    	}
    	defer f.Close()
    
    	scanner := bufio.NewScanner(f)
    	for scanner.Scan() {
    		line := scanner.Text()
    		parts := strings.SplitN(line, "\t", 2) // Assuming tab-separated input-output pairs
    		if len(parts) == 2 {
    			data = append(data, ParsedData{
    				Input:  parts[0],
    				Output: parts[1],
    			})
    		}
    	}
    	if err := scanner.Err(); err != nil {
    		return nil, err
    	}
    	return data, nil
    }
    
    func tokenize(text string) []string {
    	reg, err := regexp.Compile("[^a-zA-Z0-9А-Яа-я]+")
    	if err != nil {
    		log.Fatalf("Regex compilation failed: %v", err)
    	}
    	cleanedText := reg.ReplaceAllString(text, " ")
    	return strings.Fields(strings.ToLower(cleanedText))
    }
    
    func vectorize(tokens []string, vocab map[string]int) []float32 {
    	vector := make([]float32, len(vocab))
    	for _, token := range tokens {
    		if idx, exists := vocab[token]; exists {
    			vector[idx] += 1
    		}
    	}
    	return vector
    }
    
    func validateModel(validationDir string) {
    	weightsTensor, err := LoadWeights("model_weights.gob")
    	if err != nil {
    		log.Fatalf("Error loading model weights: %v", err)
    	}
    
    	vocab, err := LoadVocabulary("vocab.gob")
    	if err != nil {
    		log.Fatalf("Error loading vocabulary: %v", err)
    	}
    
    	validationData, err := LoadFilesRecursively(validationDir)
    	if err != nil {
    		log.Fatalf("Error loading validation data: %v", err)
    	}
    
    	g := gorgonia.NewGraph()
    	w := gorgonia.NewMatrix(g, tensor.Float32, gorgonia.WithShape(len(vocab), 1), gorgonia.WithValue(weightsTensor))
    
    	correct := 0
    	total := len(validationData)
    
    	for _, item := range validationData {
    		inputTokens := tokenize(item.Input)
    		inputVec := vectorize(inputTokens, vocab)
    
    		xVal := tensor.New(tensor.WithShape(1, len(vocab)), tensor.Of(tensor.Float32))
    		copy(xVal.Data().([]float32), inputVec)
    
    		x := gorgonia.NewMatrix(g, tensor.Float32, gorgonia.WithShape(1, len(vocab)), gorgonia.WithValue(xVal))
    		yhat := gorgonia.Must(gorgonia.Mul(x, w))
    
    		machine := gorgonia.NewTapeMachine(g)
    		if err := machine.RunAll(); err != nil {
    			log.Fatalf("Machine run error: %v", err)
    		}
    		defer machine.Close()
    
    		predictedOutput := yhat.Value().Data().([]float32)[0]
    		expectedOutput := interpretOutput(item.Output)
    
    		if predictedOutput == expectedOutput {
    			correct++
    		}
    		machine.Reset()
    	}
    
    	accuracy := float64(correct) / float64(total) * 100
    	fmt.Printf("Validation Accuracy: %.2f%%\n", accuracy)
    }
    
    func interpretOutput(output string) float32 {
    	// Placeholder: interpret output as needed or expected. Modify the logic here based on your target labels.
    	return float32(output[0])
    }
    
    func main() {
    	fmt.Print("Enter mode of operation (t for training, v for validation, d for dialogue): ")
    	scanner := bufio.NewScanner(os.Stdin)
    	scanner.Scan()
    	mode := scanner.Text()
    
    	switch mode {
    	case "t":
    		trainModel("train", maxTrainMessages)
    	case "v":
    		validateModel("valid")
    	case "d":
    		interactWithModel()
    	default:
    		fmt.Println("Unknown mode. Use 't' for training, 'v' for validation, or 'd' for dialogue.")
    	}
    }
    

    Key Changes:

    • Recursive File Loading: The LoadFilesRecursively function uses filepath.Walk to traverse directories and load all eligible files.
    • File Handlers: Specific functions loadCSVFile and loadTXTFile handle each file type.
    • Flexible Validation Processing: Ensures that every file's data is tokenized and vectorized before running through the model for validation.

    Make sure to adapt interpretOutput to match the logic of how your model training represented and expected output data. The placeholder logic compares direct values which might not be directly applicable based on your actual dataset setup.

  5. User

    нет напиши скрипть целиком и полностью

  6. Assistant

    Certainly! Below is a full Go script that incorporates recursive directory traversal for loading CSV and TXT files, applies consistent tokenization and vectorization, and evaluates a model trained with Gorgonia against validation data.

    package main
    
    import (
    	"bufio"
    	"encoding/csv"
    	"encoding/gob"
    	"fmt"
    	"log"
    	"os"
    	"path/filepath"
    	"regexp"
    	"strings"
    
    	"gorgonia.org/gorgonia"
    	"gorgonia.org/tensor"
    )
    
    type ParsedData struct {
    	Input  string
    	Output string
    }
    
    func LoadFilesRecursively(dirname string) ([]ParsedData, error) {
    	var data []ParsedData
    	err := filepath.Walk(dirname, func(path string, info os.FileInfo, err error) error {
    		if err != nil {
    			return err
    		}
    
    		if !info.IsDir() {
    			switch filepath.Ext(path) {
    			case ".csv":
    				fileData, err := loadCSVFile(path)
    				if err != nil {
    					return err
    				}
    				data = append(data, fileData...)
    			case ".txt":
    				fileData, err := loadTXTFile(path)
    				if err != nil {
    					return err
    				}
    				data = append(data, fileData...)
    			}
    		}
    		return nil
    	})
    	if err != nil {
    		return nil, err
    	}
    	return data, nil
    }
    
    func loadCSVFile(filepath string) ([]ParsedData, error) {
    	var data []ParsedData
    	f, err := os.Open(filepath)
    	if err != nil {
    		return nil, err
    	}
    	defer f.Close()
    
    	reader := csv.NewReader(f)
    	records, err := reader.ReadAll()
    	if err != nil {
    		return nil, err
    	}
    
    	for i := 0; i < len(records); i++ {
    		record := records[i]
    		if len(record) >= 2 {
    			data = append(data, ParsedData{
    				Input:  record[0],
    				Output: record[1],
    			})
    		}
    	}
    	return data, nil
    }
    
    func loadTXTFile(filepath string) ([]ParsedData, error) {
    	var data []ParsedData
    	f, err := os.Open(filepath)
    	if err != nil {
    		return nil, err
    	}
    	defer f.Close()
    
    	scanner := bufio.NewScanner(f)
    	for scanner.Scan() {
    		line := scanner.Text()
    		parts := strings.SplitN(line, "\t", 2)
    		if len(parts) == 2 {
    			data = append(data, ParsedData{
    				Input:  parts[0],
    				Output: parts[1],
    			})
    		}
    	}
    	if err := scanner.Err(); err != nil {
    		return nil, err
    	}
    	return data, nil
    }
    
    func Tokenize(text string) []string {
    	reg, err := regexp.Compile("[^a-zA-Z0-9А-Яа-я]+")
    	if err != nil {
    		log.Fatalf("Regex compilation failed: %v", err)
    	}
    	cleanedText := reg.ReplaceAllString(text, " ")
    	return strings.Fields(strings.ToLower(cleanedText))
    }
    
    func Vectorize(tokens []string, vocab map[string]int) []float32 {
    	vector := make([]float32, len(vocab))
    	for _, token := range tokens {
    		if idx, exists := vocab[token]; exists {
    			vector[idx] += 1
    		}
    	}
    	return vector
    }
    
    func LoadWeights(filename string) (*tensor.Dense, error) {
    	f, err := os.Open(filename)
    	if err != nil {
    		return nil, err
    	}
    	defer f.Close()
    
    	var weights tensor.Dense
    	decoder := gob.NewDecoder(f)
    	if err := decoder.Decode(&weights); err != nil {
    		return nil, err
    	}
    	return &weights, nil
    }
    
    func LoadVocabulary(filename string) (map[string]int, error) {
    	f, err := os.Open(filename)
    	if err != nil {
    		return nil, err
    	}
    	defer f.Close()
    
    	var vocab map[string]int
    	decoder := gob.NewDecoder(f)
    	if err := decoder.Decode(&vocab); err != nil {
    		return nil, err
    	}
    	return vocab, nil
    }
    
    func validateModel(validationDir string) {
    	weightsTensor, err := LoadWeights("model_weights.gob")
    	if err != nil {
    		log.Fatalf("Error loading model weights: %v", err)
    	}
    
    	vocab, err := LoadVocabulary("vocab.gob")
    	if err != nil {
    		log.Fatalf("Error loading vocabulary: %v", err)
    	}
    
    	validationData, err := LoadFilesRecursively(validationDir)
    	if err != nil {
    		log.Fatalf("Error loading validation data: %v", err)
    	}
    
    	g := gorgonia.NewGraph()
    	w := gorgonia.NewMatrix(g, tensor.Float32, gorgonia.WithShape(len(vocab), 1), gorgonia.WithValue(weightsTensor))
    
    	correct := 0
    	total := len(validationData)
    
    	for _, item := range validationData {
    		inputTokens := Tokenize(item.Input)
    		inputVec := Vectorize(inputTokens, vocab)
    
    		xVal := tensor.New(tensor.WithShape(1, len(vocab)), tensor.Of(tensor.Float32))
    		copy(xVal.Data().([]float32), inputVec)
    
    		x := gorgonia.NewMatrix(g, tensor.Float32, gorgonia.WithShape(1, len(vocab)), gorgonia.WithValue(xVal))
    		yhat := gorgonia.Must(gorgonia.Mul(x, w))
    
    		machine := gorgonia.NewTapeMachine(g)
    		if err := machine.RunAll(); err != nil {
    			log.Fatalf("Machine run error: %v", err)
    		}
    		defer machine.Close()
    
    		predictedOutput := yhat.Value().Data().([]float32)[0]
    		expectedOutput := interpretOutput(item.Output)
    
    		if predictedOutput == expectedOutput {
    			correct++
    		}
    		machine.Reset()
    	}
    
    	accuracy := float64(correct) / float64(total) * 100
    	fmt.Printf("Validation Accuracy: %.2f%%\n", accuracy)
    }
    
    func interpretOutput(output string) float32 {
    	// Customize this logic based on your label encoding
    	return float32(output[0])
    }
    
    func main() {
    	fmt.Print("Enter mode of operation (t for training, v for validation, d for dialogue): ")
    	scanner := bufio.NewScanner(os.Stdin)
    	scanner.Scan()
    	mode := scanner.Text()
    
    	switch mode {
    	case "t":
    		trainModel("train", maxTrainMessages)
    	case "v":
    		validateModel("valid")
    	case "d":
    		interactWithModel()
    	default:
    		fmt.Println("Unknown mode. Use 't' for training, 'v' for validation, or 'd' for dialogue.")
    	}
    }
    

    Key Components Explained:

    • Recursive File Loading: Uses filepath.Walk to find all CSV and TXT files within the specified directory and subdirectories.
    • File Handlers: Implements separate logic to load data from CSV (loadCSVFile) and TXT (loadTXTFile) files.
    • Tokenization and Vectorization: Ensures text data is consistently preprocessed into tokens and vectors using the trained vocabulary.
    • Model Validation: Loads model weights and vocabulary, then calculates accuracy over validation data.

    Customization Needed:

    • interpretOutput Function: Adjust how the expected output is interpreted and compared to predictions to suit your label format.
    • Training Module: Ensure the trainModel function and vocabulary/weights saving logic are implemented similar to the earlier code for completeness.

    Note:

    • The interpretOutput function should map output labels from text to whatever format the model expects, altering its logic based on your specific dataset.
    • This script assumes a simplistic numeric interpretation of labels; if your outputs are more complex, you'll need additional parsing and processing.
  7. User

    верни train и interactWithModel

  8. Assistant

    Certainly! I'll provide you with the complete script, including the trainModel and interactWithModel functions, along with handling the recursive loading of CSV and TXT files. This version of the script ensures that both training and interaction with the model can be performed effectively.

    package main
    
    import (
    	"bufio"
    	"encoding/csv"
    	"encoding/gob"
    	"fmt"
    	"log"
    	"os"
    	"path/filepath"
    	"regexp"
    	"sort"
    	"strings"
    
    	"gorgonia.org/gorgonia"
    	"gorgonia.org/tensor"
    )
    
    type ParsedData struct {
    	Input  string
    	Output string
    }
    
    const maxTrainMessages = 10
    const maxVocabSize = 500000
    
    func LoadFilesRecursively(dirname string) ([]ParsedData, error) {
    	var data []ParsedData
    	err := filepath.Walk(dirname, func(path string, info os.FileInfo, err error) error {
    		if err != nil {
    			return err
    		}
    
    		if !info.IsDir() {
    			switch filepath.Ext(path) {
    			case ".csv":
    				fileData, err := loadCSVFile(path)
    				if err != nil {
    					return err
    				}
    				data = append(data, fileData...)
    			case ".txt":
    				fileData, err := loadTXTFile(path)
    				if err != nil {
    					return err
    				}
    				data = append(data, fileData...)
    			}
    		}
    		return nil
    	})
    	if err != nil {
    		return nil, err
    	}
    	return data, nil
    }
    
    func loadCSVFile(filepath string) ([]ParsedData, error) {
    	var data []ParsedData
    	f, err := os.Open(filepath)
    	if err != nil {
    		return nil, err
    	}
    	defer f.Close()
    
    	reader := csv.NewReader(f)
    	records, err := reader.ReadAll()
    	if err != nil {
    		return nil, err
    	}
    
    	for i := 0; i < len(records); i++ {
    		record := records[i]
    		if len(record) >= 2 {
    			data = append(data, ParsedData{
    				Input:  record[0],
    				Output: record[1],
    			})
    		}
    	}
    	return data, nil
    }
    
    func loadTXTFile(filepath string) ([]ParsedData, error) {
    	var data []ParsedData
    	f, err := os.Open(filepath)
    	if err != nil {
    		return nil, err
    	}
    	defer f.Close()
    
    	scanner := bufio.NewScanner(f)
    	for scanner.Scan() {
    		line := scanner.Text()
    		parts := strings.SplitN(line, "\t", 2)
    		if len(parts) == 2 {
    			data = append(data, ParsedData{
    				Input:  parts[0],
    				Output: parts[1],
    			})
    		}
    	}
    	if err := scanner.Err(); err != nil {
    		return nil, err
    	}
    	return data, nil
    }
    
    func Tokenize(text string) []string {
    	reg, err := regexp.Compile("[^a-zA-Z0-9А-Яа-я]+")
    	if err != nil {
    		log.Fatalf("Regex compilation failed: %v", err)
    	}
    	cleanedText := reg.ReplaceAllString(text, " ")
    	return strings.Fields(strings.ToLower(cleanedText))
    }
    
    func BuildVocab(data []ParsedData) map[string]int {
    	vocab := make(map[string]int)
    	wordFreq := make(map[string]int)
    
    	for _, item := range data {
    		tokens := append(Tokenize(item.Input), Tokenize(item.Output)...)
    		for _, token := range tokens {
    			wordFreq[token]++
    		}
    	}
    
    	type kv struct {
    		Key   string
    		Value int
    	}
    
    	var freqs []kv
    	for k, v := range wordFreq {
    		freqs = append(freqs, kv{k, v})
    	}
    
    	sort.Slice(freqs, func(i, j int) bool {
    		return freqs[i].Value > freqs[j].Value
    	})
    
    	if len(freqs) > maxVocabSize {
    		freqs = freqs[:maxVocabSize]
    	}
    
    	for i, kv := range freqs {
    		vocab[kv.Key] = i
    	}
    	return vocab
    }
    
    func Vectorize(tokens []string, vocab map[string]int) []float32 {
    	vector := make([]float32, len(vocab))
    	for _, token := range tokens {
    		if idx, exists := vocab[token]; exists {
    			vector[idx] += 1
    		}
    	}
    	return vector
    }
    
    func SaveWeights(weights *tensor.Dense, filename string) error {
    	f, err := os.Create(filename)
    	if err != nil {
    		return err
    	}
    	defer f.Close()
    
    	encoder := gob.NewEncoder(f)
    	return encoder.Encode(weights)
    }
    
    func LoadWeights(filename string) (*tensor.Dense, error) {
    	f, err := os.Open(filename)
    	if err != nil {
    		return nil, err
    	}
    	defer f.Close()
    
    	var weights tensor.Dense
    	decoder := gob.NewDecoder(f)
    	if err := decoder.Decode(&weights); err != nil {
    		return nil, err
    	}
    	return &weights, nil
    }
    
    func SaveVocabulary(vocab map[string]int, filename string) error {
    	f, err := os.Create(filename)
    	if err != nil {
    		return err
    	}
    	defer f.Close()
    
    	encoder := gob.NewEncoder(f)
    	return encoder.Encode(vocab)
    }
    
    func LoadVocabulary(filename string) (map[string]int, error) {
    	f, err := os.Open(filename)
    	if err != nil {
    		return nil, err
    	}
    	defer f.Close()
    
    	var vocab map[string]int
    	decoder := gob.NewDecoder(f)
    	if err := decoder.Decode(&vocab); err != nil {
    		return nil, err
    	}
    	return vocab, nil
    }
    
    func trainModel(trainDir string, maxTrainMessages int) {
    	parsedData, err := LoadFilesRecursively(trainDir)
    	if err != nil {
    		log.Fatalf("Error loading training data: %v", err)
    	}
    
    	if maxTrainMessages > len(parsedData) {
    		maxTrainMessages = len(parsedData)
    	}
    
    	parsedData = parsedData[:maxTrainMessages]
    	fmt.Printf("Using %d training records for modeling\n", len(parsedData))
    
    	vocab := BuildVocab(parsedData)
    
    	if err := SaveVocabulary(vocab, "vocab.gob"); err != nil {
    		log.Fatalf("Error saving vocabulary: %v", err)
    	}
    
    	batchSize := 150
    	totalSamples := len(parsedData)
    	numEpochs := 10
    
    	g := gorgonia.NewGraph()
    
    	inputSize := len(vocab)
    	w := gorgonia.NewMatrix(g, tensor.Float32, gorgonia.WithShape(inputSize, 1), gorgonia.WithInit(gorgonia.GlorotN(1.0)))
    
    	optimizer := gorgonia.NewAdamSolver(gorgonia.WithLearnRate(0.001))
    
    	machine := gorgonia.NewTapeMachine(g, gorgonia.BindDualValues(w))
    	defer machine.Close()
    
    	for epoch := 0; epoch < numEpochs; epoch++ {
    		for start := 0; start < totalSamples; start += batchSize {
    			end := start + batchSize
    			if end > totalSamples {
    				end = totalSamples
    			}
    
    			batchSizeActual := end - start
    			xVal := tensor.New(tensor.WithShape(batchSizeActual, inputSize), tensor.Of(tensor.Float32))
    			yVal := tensor.New(tensor.WithShape(batchSizeActual, 1), tensor.Of(tensor.Float32))
    
    			for i := start; i < end; i++ {
    				tokens := Tokenize(parsedData[i].Input)
    				inputVec := Vectorize(tokens, vocab)
    				offset := (i - start) * inputSize
    				copy(xVal.Data().([]float32)[offset:], inputVec)
    
    				outputValue := float32(0)
    				if parsedData[i].Output != "" {
    					outputValue = float32(parsedData[i].Output[0])
    				}
    				yVal.SetAt(outputValue, i-start, 0)
    			}
    
    			x := gorgonia.NewMatrix(g, tensor.Float32, gorgonia.WithShape(batchSizeActual, inputSize), gorgonia.WithValue(xVal))
    			y := gorgonia.NewMatrix(g, tensor.Float32, gorgonia.WithShape(batchSizeActual, 1), gorgonia.WithValue(yVal))
    
    			yhat := gorgonia.Must(gorgonia.Mul(x, w))
    			loss := gorgonia.Must(gorgonia.Mean(gorgonia.Must(gorgonia.Square(gorgonia.Must(gorgonia.Sub(yhat, y))))))
    
    			if _, err := gorgonia.Grad(loss, w); err != nil {
    				log.Fatalf("Failed to create computation graph: %v", err)
    			}
    
    			if err := machine.RunAll(); err != nil {
    				log.Fatalf("Machine run error: %v", err)
    			}
    
    			optimizer.Step(gorgonia.NodesToValueGrads([]*gorgonia.Node{w}))
    
    			machine.Reset()
    
    			totalBatches := (totalSamples + batchSize - 1) / batchSize
    			totalProgress := float64(epoch*totalBatches+start/batchSize+1) / float64(totalBatches*numEpochs) * 100
    			progress := float64(end) / float64(totalSamples) * 100
    			fmt.Printf("Epoch %d: Processed batch from %d to %d (%.2f%% Epoch complete)(%.2f%% complete overall)\n", epoch+1, start, end, progress, totalProgress)
    		}
    	}
    
    	fmt.Println("Model training complete")
    
    	if err := SaveWeights(w.Value().(*tensor.Dense), "model_weights.gob"); err != nil {
    		log.Fatalf("Error saving model weights: %v", err)
    	} else {
    		fmt.Println("Model weights saved successfully.")
    	}
    }
    
    func interactWithModel() {
    	weightsTensor, err := LoadWeights("model_weights.gob")
    	if err != nil {
    		log.Fatalf("Error loading model weights: %v", err)
    	}
    
    	vocab, err := LoadVocabulary("vocab.gob")
    	if err != nil {
    		log.Fatalf("Error loading vocabulary: %v", err)
    	}
    
    	reverseVocab := make(map[int]string)
    	for word, idx := range vocab {
    		reverseVocab[idx] = word
    	}
    
    	inputSize := len(vocab)
    	reader := bufio.NewReader(os.Stdin)
    
    	g := gorgonia.NewGraph()
    	w := gorgonia.NewMatrix(g, tensor.Float32, gorgonia.WithShape(inputSize, 1), gorgonia.WithValue(weightsTensor))
    
    	for {
    		fmt.Print("Ask a question (or type 'exit' to quit): ")
    		text, _ := reader.ReadString('\n')
    		text = strings.TrimSpace(text)
    
    		if text == "exit" {
    			break
    		}
    
    		tokens := Tokenize(text)
    		inputVec := Vectorize(tokens, vocab)
    
    		xVal := tensor.New(tensor.WithShape(1, inputSize), tensor.Of(tensor.Float32))
    		copy(xVal.Data().([]float32), inputVec)
    
    		x := gorgonia.NewMatrix(g, tensor.Float32, gorgonia.WithShape(1, inputSize), gorgonia.WithValue(xVal))
    		yhat := gorgonia.Must(gorgonia.Mul(x, w))
    
    		machine := gorgonia.NewTapeMachine(g)
    		defer machine.Close()
    
    		if err := machine.RunAll(); err != nil {
    			log.Fatalf("Machine run error: %v", err)
    		}
    
    		predictionsVal := yhat.Value().Data().([]float32)
    		topIndices := getTopIndices(predictionsVal, 5)
    
    		var responseTokens []string
    		for _, idx := range topIndices {
    			if word, exists := reverseVocab[idx]; exists {
    				responseTokens = append(responseTokens, word)
    			}
    		}
    
    		response := strings.Join(responseTokens, " ")
    		fmt.Printf("Generated Response: %s\n", response)
    		machine.Reset()
    	}
    }
    
    func getTopIndices(values []float32, n int) []int {
    	type kv struct {
    		Key   int
    		Value float32
    	}
    
    	var ss []kv
    	for i, val := range values {
    		ss = append(ss, kv{i, val})
    	}
    
    	sort.Slice(ss, func(i, j int) bool {
    		return ss[i].Value > ss[j].Value
    	})
    
    	var indices []int
    	for i := 0; i < n && i < len(ss); i++ {
    		indices = append(indices, ss[i].Key)
    	}
    	return indices
    }
    
    func main() {
    	fmt.Print("Enter mode of operation (t for training, v for validation, d for dialogue): ")
    	scanner := bufio.NewScanner(os.Stdin)
    	scanner.Scan()
    	mode := scanner.Text()
    
    	switch mode {
    	case "t":
    		trainModel("train", maxTrainMessages)
    	case "v":
    		validateModel("valid")
    	case "d":
    		interactWithModel()
    	default:
    		fmt.Println("Unknown mode. Use 't' for training, 'v' for validation, or 'd' for dialogue.")
    	}
    }
    

    Explanation:

    • Training Mode (trainModel): It reads data from specified directory, builds a vocabulary, vectorizes the data, trains a linear model, and saves the trained model's weights.
    • Dialogue Mode (interactWithModel): Reads user input, vectorizes it using the vocabulary, and generates a response by passing it through the trained model.
    • Recursive Directory Traversal: Handles loading of CSV and TXT files from potentially nested directories for training and validation data.
    • Model Validation and Interaction: Uses pre-trained weights and vocabulary for validation and interaction.

    Note: Adjust interpretOutput and input/output handling according to your dataset and model's expected data format. This script assumes basic numeric handling of outputs for simplicity, which might need to be expanded for more complex datasets.