Skip to content
TheChatArchiveThe Chat Archive
o1-mini-2024-09-12· OpenAIAi2 WildChat-4.8M (AI2)

package main import ( "bufio" "encoding/csv" "encoding/gob" "fmt" "io/ioutil" "log"…

  1. User

    package main

    import ( "bufio" "encoding/csv" "encoding/gob" "fmt" "io/ioutil" "log" "math" "os" "path/filepath" "sort" "strings"

    "gorgonia.org/gorgonia"
    "gorgonia.org/tensor"
    

    )

    // ParsedData представляет структуру для входных и выходных данных type ParsedData struct { Input string Output string }

    // Константы для ограничения количества сообщений и размера словаря const ( maxTrainMessages = 10000 maxVocabSize = 500000 )

    // LoadFiles загружает файлы из указанной директории func LoadFiles(dirname string, maxMessages int) ([]ParsedData, error) { var data []ParsedData err := filepath.WalkDir(dirname, func(path string, d os.DirEntry, err error) error { if err != nil { return err } if d.IsDir() || len(data) >= maxMessages { return nil }

    	if strings.HasSuffix(d.Name(), ".csv") {
    		f, err := os.Open(path)
    		if err != nil {
    			return err
    		}
    		defer f.Close()
    
    		r := csv.NewReader(f)
    		records, err := r.ReadAll()
    		if err != nil {
    			return err
    		}
    
    		for i := 1; i < len(records) && len(data) < maxMessages; i++ { // Предполагается, что первая строка - заголовок
    			record := records[i]
    			if len(record) >= 2 {
    				data = append(data, ParsedData{
    					Input:  record[0],
    					Output: record[1],
    				})
    			}
    		}
    	} else if strings.HasSuffix(d.Name(), ".txt") {
    		content, err := ioutil.ReadFile(path)
    		if err != nil {
    			return err
    		}
    		lines := strings.Split(string(content), "\n")
    		for _, line := range lines {
    			if len(data) >= maxMessages {
    				break
    			}
    			parts := strings.Split(line, ":")
    			if len(parts) > 1 {
    				data = append(data, ParsedData{
    					Input:  strings.TrimSpace(parts[0]),
    					Output: strings.TrimSpace(parts[1]),
    				})
    			}
    		}
    	}
    	return nil
    })
    return data, err
    

    }

    // Tokenize разбивает текст на токены func Tokenize(text string) []string { return strings.Fields(strings.TrimSpace(text)) }

    // BuildVocab строит словарь из набора данных func BuildVocab(data []ParsedData) map[string]int { vocab := make(map[string]int) wordFreq := make(map[string]int)

    for _, item := range data {
    	tokens := append(Tokenize(item.Input), Tokenize(item.Output)...)
    	for _, token := range tokens {
    		wordFreq[token]++
    	}
    }
    
    type kv struct {
    	Key   string
    	Value int
    }
    
    var freqs []kv
    for k, v := range wordFreq {
    	freqs = append(freqs, kv{k, v})
    }
    
    sort.Slice(freqs, func(i, j int) bool {
    	return freqs[i].Value > freqs[j].Value
    })
    
    if len(freqs) > maxVocabSize {
    	freqs = freqs[:maxVocabSize]
    }
    
    for i, kv := range freqs {
    	vocab[kv.Key] = i
    }
    return vocab
    

    }

    // Vectorize преобразует токены в вектор признаков func Vectorize(tokens []string, vocab map[string]int) []float32 { vector := make([]float32, len(vocab)) for _, token := range tokens { if idx, exists := vocab[token]; exists { vector[idx] += 1 } } return vector }

    // Сохранение и загрузка весов модели func SaveWeights(weights *tensor.Dense, filename string) error { f, err := os.Create(filename) if err != nil { return err } defer f.Close()

    encoder := gob.NewEncoder(f)
    return encoder.Encode(weights)
    

    }

    func LoadWeights(filename string) (*tensor.Dense, error) { f, err := os.Open(filename) if err != nil { return nil, err } defer f.Close()

    var weights tensor.Dense
    decoder := gob.NewDecoder(f)
    if err := decoder.Decode(&weights); err != nil {
    	return nil, err
    }
    return &weights, nil
    

    }

    // Сохранение и загрузка словаря func SaveVocabulary(vocab map[string]int, filename string) error { f, err := os.Create(filename) if err != nil { return err } defer f.Close()

    encoder := gob.NewEncoder(f)
    return encoder.Encode(vocab)
    

    }

    func LoadVocabulary(filename string) (map[string]int, error) { f, err := os.Open(filename) if err != nil { return nil, err } defer f.Close()

    var vocab map[string]int
    decoder := gob.NewDecoder(f)
    if err := decoder.Decode(&vocab); err != nil {
    	return nil, err
    }
    return vocab, nil
    

    }

    // sigmoid применяет сигмоиду к каждому элементу тензора func sigmoid(x tensor.Tensor) tensor.Tensor { data := x.Data().([]float32) result := make([]float32, len(data)) for i, v := range data { result[i] = 1 / (1 + float32(math.Exp(-float64(v)))) } return tensor.New(tensor.Of(tensor.Float32), tensor.WithBacking(result), tensor.WithShape(x.Shape()...)) }

    // trainModel обучает модель на данных из trainDir func trainModel(trainDir string, maxTrainMessages int) { parsedData, err := LoadFiles(trainDir, maxTrainMessages) if err != nil { log.Fatalf("Error loading training files: %v", err) }

    fmt.Printf("Loaded %d training records\n", len(parsedData))
    
    vocab := BuildVocab(parsedData)
    
    // Create models directory if it doesn't exist
    if err := os.MkdirAll("models", os.ModePerm); err != nil {
    	log.Fatalf("Error creating models directory: %v", err)
    }
    
    // Save vocabulary to models directory
    if err := SaveVocabulary(vocab, "models/vocab.gob"); err != nil {
    	log.Fatalf("Error saving vocabulary: %v", err)
    }
    
    batchSize := 150
    totalSamples := len(parsedData)
    numEpochs := 10
    
    g := gorgonia.NewGraph()
    
    inputSize := len(vocab)
    w := gorgonia.NewMatrix(g, tensor.Float32, gorgonia.WithShape(inputSize, 1), gorgonia.WithInit(gorgonia.GlorotN(1.0)))
    
    x := gorgonia.NewMatrix(g, tensor.Float32, gorgonia.WithShape(batchSize, inputSize), gorgonia.WithName("x"))
    y := gorgonia.NewMatrix(g, tensor.Float32, gorgonia.WithShape(batchSize, 1), gorgonia.WithName("y"))
    
    yhat := gorgonia.Must(gorgonia.Mul(x, w))
    yhatSig := gorgonia.Must(gorgonia.Sigmoid(yhat)) // Apply sigmoid
    
    // Binary cross-entropy loss
    losses := gorgonia.Must(gorgonia.Sub(yhatSig, y))
    squaredLoss := gorgonia.Must(gorgonia.Square(losses))
    loss := gorgonia.Must(gorgonia.Mean(squaredLoss))
    
    // Create gradients
    _, err = gorgonia.Grad(loss, w)
    if err != nil {
    	log.Fatalf("Failed to create gradients: %v", err)
    }
    
    // Initialize optimizer
    optimizer := gorgonia.NewAdamSolver(gorgonia.WithLearnRate(0.001))
    
    // Create the machine
    machine := gorgonia.NewTapeMachine(g, gorgonia.BindDualValues(w))
    defer machine.Close()
    
    for epoch := 0; epoch < numEpochs; epoch++ {
    	var epochLoss float64
    	for start := 0; start < totalSamples; start += batchSize {
    		end := start + batchSize
    		if end > totalSamples {
    			end = totalSamples
    		}
    
    		batchSizeActual := end - start
    		xValData := make([]float32, batchSizeActual*inputSize)
    		yValData := make([]float32, batchSizeActual)
    
    		for i := start; i < end; i++ {
    			tokens := Tokenize(parsedData[i].Input)
    			inputVec := Vectorize(tokens, vocab)
    			copy(xValData[(i-start)*inputSize:(i-start+1)*inputSize], inputVec)
    
    			// Parse Output as float32
    			if parsedData[i].Output == "1" {
    				yValData[i-start] = 1.0
    			} else {
    				yValData[i-start] = 0.0
    			}
    		}
    
    		xVal := tensor.New(tensor.WithShape(batchSizeActual, inputSize), tensor.Of(tensor.Float32), tensor.WithBacking(xValData))
    		yVal := tensor.New(tensor.WithShape(batchSizeActual, 1), tensor.Of(tensor.Float32), tensor.WithBacking(yValData))
    
    		// Set node values using gorgonia.Let
    		if err := gorgonia.Let(x, xVal); err != nil {
    			log.Fatalf("Error setting x value: %v", err)
    		}
    		if err := gorgonia.Let(y, yVal); err != nil {
    			log.Fatalf("Error setting y value: %v", err)
    		}
    
    		if err := machine.RunAll(); err != nil {
    			log.Fatalf("Machine run error during training: %v", err)
    		}
    
    		// Get loss value
    		lossData := loss.Value().Data()
    		lossVal, ok := lossData.(float32)
    		if !ok {
    			log.Fatalf("Unexpected type for loss value: %T", lossData)
    		}
    
    		epochLoss += float64(lossVal) * float64(batchSizeActual)
    
    		// Update weights
    		nodes := gorgonia.Nodes{w}
    		err = optimizer.Step(gorgonia.NodesToValueGrads(nodes))
    		if err != nil {
    			log.Fatalf("Error during optimizer step: %v", err)
    		}
    
    		machine.Reset()
    	}
    
    	averageLoss := epochLoss / float64(totalSamples)
    	fmt.Printf("Epoch %d/%d - Loss: %.4f\n", epoch+1, numEpochs, averageLoss)
    }
    
    fmt.Println("Model training complete")
    
    // Save weights to models directory
    if err := SaveWeights(w.Value().(*tensor.Dense), "models/model_weights.gob"); err != nil {
    	log.Fatalf("Error saving model weights: %v", err)
    } else {
    	fmt.Println("Model weights saved successfully.")
    }
    

    }

    // interactWithModel позволяет взаимодействовать с обученной моделью func interactWithModel() { weightsTensor, err := LoadWeights("models/model_weights.gob") if err != nil { log.Fatalf("Error loading model weights: %v", err) }

    vocab, err := LoadVocabulary("models/vocab.gob")
    if err != nil {
    	log.Fatalf("Error loading vocabulary: %v", err)
    }
    
    inputSize := len(vocab)
    reader := bufio.NewReader(os.Stdin)
    
    // Создаем граф для предсказания
    g := gorgonia.NewGraph()
    w := gorgonia.NewMatrix(g, tensor.Float32, gorgonia.WithShape(inputSize, 1), gorgonia.WithValue(weightsTensor))
    
    x := gorgonia.NewMatrix(g, tensor.Float32, gorgonia.WithShape(1, inputSize))
    yhat := gorgonia.Must(gorgonia.Mul(x, w))
    yhatSig := gorgonia.Must(gorgonia.Sigmoid(yhat)) // Применяем сигмоиду
    
    machine := gorgonia.NewTapeMachine(g, gorgonia.BindDualValues(w))
    defer machine.Close()
    
    for {
    	fmt.Print("Ask a question (or type 'exit' to quit): ")
    	text, _ := reader.ReadString('\n')
    	text = strings.TrimSpace(text)
    
    	if text == "exit" {
    		break
    	}
    
    	tokens := Tokenize(text)
    	inputVec := Vectorize(tokens, vocab)
    
    	xValData := make([]float32, inputSize)
    	copy(xValData, inputVec)
    	xVal := tensor.New(tensor.WithShape(1, inputSize), tensor.Of(tensor.Float32), tensor.WithBacking(xValData))
    
    	// Устанавливаем значение узла x с помощью gorgonia.Let
    	if err := gorgonia.Let(x, xVal); err != nil {
    		log.Fatalf("Error setting x value: %v", err)
    	}
    
    	if err := machine.RunAll(); err != nil {
    		log.Fatalf("Machine run error during interaction: %v", err)
    	}
    
    	predictionsVal := yhatSig.Value().Data().([]float32)
    	predictedProbability := predictionsVal[0]
    	predictedOutput := 0
    	if predictedProbability >= 0.5 {
    		predictedOutput = 1
    	}
    
    	var response string
    	if predictedOutput == 1 {
    		response = "Positive Response"
    	} else {
    		response = "Negative Response"
    	}
    
    	fmt.Printf("Generated Response: %s (Probability: %.4f)\n", response, predictedProbability)
    	machine.Reset()
    }
    

    }

    // validateModel выполняет валидацию модели на данных из validationDir func validateModel(validationDir string) { weightsTensor, err := LoadWeights("models/model_weights.gob") if err != nil { log.Fatalf("Error loading model weights: %v", err) }

    vocab, err := LoadVocabulary("models/vocab.gob")
    if err != nil {
    	log.Fatalf("Error loading vocabulary: %v", err)
    }
    
    validationData, err := LoadFiles(validationDir, maxTrainMessages) // Используйте подходящее ограничение
    if err != nil {
    	log.Fatalf("Error loading validation files: %v", err)
    }
    
    fmt.Printf("Loaded %d validation records\n", len(validationData))
    
    g := gorgonia.NewGraph()
    w := gorgonia.NewMatrix(g, tensor.Float32, gorgonia.WithShape(len(vocab), 1), gorgonia.WithValue(weightsTensor))
    
    x := gorgonia.NewMatrix(g, tensor.Float32, gorgonia.WithShape(1, len(vocab)))
    y := gorgonia.NewMatrix(g, tensor.Float32, gorgonia.WithShape(1, 1))
    
    yhat := gorgonia.Must(gorgonia.Mul(x, w))
    yhatSig := gorgonia.Must(gorgonia.Sigmoid(yhat)) // Применяем сигмоиду
    
    // Валидация не требует градиентов, поэтому не нужно вызывать Grad
    machine := gorgonia.NewTapeMachine(g, gorgonia.BindDualValues(w))
    defer machine.Close()
    
    var totalLoss float64
    correct := 0
    total := len(validationData)
    
    for idx, item := range validationData {
    	inputTokens := Tokenize(item.Input)
    	inputVec := Vectorize(inputTokens, vocab)
    
    	xValData := make([]float32, len(vocab))
    	copy(xValData, inputVec)
    	xVal := tensor.New(tensor.WithShape(1, len(vocab)), tensor.Of(tensor.Float32), tensor.WithBacking(xValData))
    
    	// Парсим Output как float32
    	var expectedOutput float32
    	if item.Output == "1" {
    		expectedOutput = 1.0
    	} else {
    		expectedOutput = 0.0
    	}
    
    	yValData := []float32{expectedOutput}
    	yVal := tensor.New(tensor.WithShape(1, 1), tensor.Of(tensor.Float32), tensor.WithBacking(yValData))
    
    	// Устанавливаем значения узлов x и y с помощью gorgonia.Let
    	if err := gorgonia.Let(x, xVal); err != nil {
    		log.Fatalf("Error setting x value: %v", err)
    	}
    	if err := gorgonia.Let(y, yVal); err != nil {
    		log.Fatalf("Error setting y value: %v", err)
    	}
    
    	if err := machine.RunAll(); err != nil {
    		log.Fatalf("Machine run error during validation: %v", err)
    	}
    
    	predictionsVal := yhatSig.Value().Data().([]float32)
    	predictedProbability := predictionsVal[0]
    	predictedOutput := 0
    	if predictedProbability >= 0.5 {
    		predictedOutput = 1
    	}
    
    	// Вычисляем бинарную кросс-энтропию для текущего примера
    	epsilon := float32(1e-7) // Для предотвращения log(0)
    	p := float32(math.Min(math.Max(float64(predictedProbability), float64(epsilon)), 1-1e-7))
    	lossVal := -expectedOutput*float32(math.Log(float64(p))) - (1-expectedOutput)*float32(math.Log(float64(1-p)))
    	totalLoss += float64(lossVal)
    
    	if predictedOutput == int(expectedOutput) {
    		correct++
    	}
    
    	machine.Reset()
    
    	// Опционально: вывод прогресса
    	if (idx+1)%1000 == 0 || idx+1 == total {
    		fmt.Printf("Validated %d/%d examples\n", idx+1, total)
    	}
    }
    
    averageLoss := totalLoss / float64(total)
    accuracy := float64(correct) / float64(total) * 100
    fmt.Printf("Validation Loss: %.4f\n", averageLoss)
    fmt.Printf("Validation Accuracy: %.2f%%\n", accuracy)
    

    }

    // main - точка входа программы func main() { fmt.Print("Enter mode of operation (t for training, v for validation, d for dialogue): ") scanner := bufio.NewScanner(os.Stdin) scanner.Scan() mode := scanner.Text()

    switch mode {
    case "t":
    	trainModel("train", maxTrainMessages)
    case "v":
    	validateModel("valid")
    case "d":
    	interactWithModel()
    default:
    	fmt.Println("Unknown mode. Use 't' for training, 'v' for validation, or 'd' for dialogue.")
    }
    

    }

    сделай так чтобы он выдавал не Generated Response: Positive Response (Probability: 0.5000) а внятный ответ из буквс симбвалос и остапльного ответ ии должен составляться на базе обучнеий ии сам должен его составлять на базе обучненний напиши скрипт полностью абсолютно все функции и чтобы все методы и функции существовали и типы были правильные и чтобы все что уже есть ты не удалял

    сделай так чтобы ответ был хотябы немного развернутый а не просто одно слово напиши код целиком и понлостью и может быть

  2. Assistant

    Certainly! To enhance your existing Go program so that it generates more meaningful and dynamic responses based on the training data, we can modify the interactWithModel function. Instead of returning fixed responses like "Positive Response" or "Negative Response," the program will now select appropriate responses from the training data based on the model's prediction.

    Here's the complete updated script with all necessary functions and modifications:

    package main
    
    import (
    	"bufio"
    	"encoding/csv"
    	"encoding/gob"
    	"fmt"
    	"io/ioutil"
    	"log"
    	"math"
    	"math/rand"
    	"os"
    	"path/filepath"
    	"sort"
    	"strings"
    	"time"
    
    	"gorgonia.org/gorgonia"
    	"gorgonia.org/tensor"
    )
    
    // ParsedData represents the structure for input and output data
    type ParsedData struct {
    	Input  string
    	Output string
    }
    
    // Constants to limit the number of messages and vocabulary size
    const (
    	maxTrainMessages = 10000
    	maxVocabSize     = 500000
    )
    
    // LoadFiles loads files from the specified directory
    func LoadFiles(dirname string, maxMessages int) ([]ParsedData, error) {
    	var data []ParsedData
    	err := filepath.WalkDir(dirname, func(path string, d os.DirEntry, err error) error {
    		if err != nil {
    			return err
    		}
    		if d.IsDir() || len(data) >= maxMessages {
    			return nil
    		}
    
    		if strings.HasSuffix(d.Name(), ".csv") {
    			f, err := os.Open(path)
    			if err != nil {
    				return err
    			}
    			defer f.Close()
    
    			r := csv.NewReader(f)
    			records, err := r.ReadAll()
    			if err != nil {
    				return err
    			}
    
    			for i := 1; i < len(records) && len(data) < maxMessages; i++ { // Assuming the first line is a header
    				record := records[i]
    				if len(record) >= 2 {
    					data = append(data, ParsedData{
    						Input:  record[0],
    						Output: record[1],
    					})
    				}
    			}
    		} else if strings.HasSuffix(d.Name(), ".txt") {
    			content, err := ioutil.ReadFile(path)
    			if err != nil {
    				return err
    			}
    			lines := strings.Split(string(content), "\n")
    			for _, line := range lines {
    				if len(data) >= maxMessages {
    					break
    				}
    				parts := strings.Split(line, ":")
    				if len(parts) > 1 {
    					data = append(data, ParsedData{
    						Input:  strings.TrimSpace(parts[0]),
    						Output: strings.TrimSpace(parts[1]),
    					})
    				}
    			}
    		}
    		return nil
    	})
    	return data, err
    }
    
    // Tokenize splits text into tokens
    func Tokenize(text string) []string {
    	return strings.Fields(strings.TrimSpace(text))
    }
    
    // BuildVocab builds a vocabulary from the dataset
    func BuildVocab(data []ParsedData) map[string]int {
    	vocab := make(map[string]int)
    	wordFreq := make(map[string]int)
    
    	for _, item := range data {
    		tokens := append(Tokenize(item.Input), Tokenize(item.Output)...)
    		for _, token := range tokens {
    			wordFreq[token]++
    		}
    	}
    
    	type kv struct {
    		Key   string
    		Value int
    	}
    
    	var freqs []kv
    	for k, v := range wordFreq {
    		freqs = append(freqs, kv{k, v})
    	}
    
    	sort.Slice(freqs, func(i, j int) bool {
    		return freqs[i].Value > freqs[j].Value
    	})
    
    	if len(freqs) > maxVocabSize {
    		freqs = freqs[:maxVocabSize]
    	}
    
    	for i, kv := range freqs {
    		vocab[kv.Key] = i
    	}
    	return vocab
    }
    
    // Vectorize converts tokens into a feature vector
    func Vectorize(tokens []string, vocab map[string]int) []float32 {
    	vector := make([]float32, len(vocab))
    	for _, token := range tokens {
    		if idx, exists := vocab[token]; exists {
    			vector[idx] += 1
    		}
    	}
    	return vector
    }
    
    // SaveWeights saves the model weights to a file
    func SaveWeights(weights *tensor.Dense, filename string) error {
    	f, err := os.Create(filename)
    	if err != nil {
    		return err
    	}
    	defer f.Close()
    
    	encoder := gob.NewEncoder(f)
    	return encoder.Encode(weights)
    }
    
    // LoadWeights loads the model weights from a file
    func LoadWeights(filename string) (*tensor.Dense, error) {
    	f, err := os.Open(filename)
    	if err != nil {
    		return nil, err
    	}
    	defer f.Close()
    
    	var weights tensor.Dense
    	decoder := gob.NewDecoder(f)
    	if err := decoder.Decode(&weights); err != nil {
    		return nil, err
    	}
    	return &weights, nil
    }
    
    // SaveVocabulary saves the vocabulary to a file
    func SaveVocabulary(vocab map[string]int, filename string) error {
    	f, err := os.Create(filename)
    	if err != nil {
    		return err
    	}
    	defer f.Close()
    
    	encoder := gob.NewEncoder(f)
    	return encoder.Encode(vocab)
    }
    
    // LoadVocabulary loads the vocabulary from a file
    func LoadVocabulary(filename string) (map[string]int, error) {
    	f, err := os.Open(filename)
    	if err != nil {
    		return nil, err
    	}
    	defer f.Close()
    
    	var vocab map[string]int
    	decoder := gob.NewDecoder(f)
    	if err := decoder.Decode(&vocab); err != nil {
    		return nil, err
    	}
    	return vocab, nil
    }
    
    // sigmoid applies the sigmoid function to each element of the tensor
    func sigmoid(x tensor.Tensor) tensor.Tensor {
    	data := x.Data().([]float32)
    	result := make([]float32, len(data))
    	for i, v := range data {
    		result[i] = 1 / (1 + float32(math.Exp(-float64(v))))
    	}
    	return tensor.New(tensor.Of(tensor.Float32), tensor.WithBacking(result), tensor.WithShape(x.Shape()...))
    }
    
    // trainModel trains the model using data from trainDir
    func trainModel(trainDir string, maxTrainMessages int) {
    	parsedData, err := LoadFiles(trainDir, maxTrainMessages)
    	if err != nil {
    		log.Fatalf("Error loading training files: %v", err)
    	}
    
    	fmt.Printf("Loaded %d training records\n", len(parsedData))
    
    	vocab := BuildVocab(parsedData)
    
    	// Create models directory if it doesn't exist
    	if err := os.MkdirAll("models", os.ModePerm); err != nil {
    		log.Fatalf("Error creating models directory: %v", err)
    	}
    
    	// Save vocabulary to models directory
    	if err := SaveVocabulary(vocab, "models/vocab.gob"); err != nil {
    		log.Fatalf("Error saving vocabulary: %v", err)
    	}
    
    	batchSize := 150
    	totalSamples := len(parsedData)
    	numEpochs := 10
    
    	g := gorgonia.NewGraph()
    
    	inputSize := len(vocab)
    	w := gorgonia.NewMatrix(g, tensor.Float32, gorgonia.WithShape(inputSize, 1), gorgonia.WithInit(gorgonia.GlorotN(1.0)))
    
    	x := gorgonia.NewMatrix(g, tensor.Float32, gorgonia.WithShape(batchSize, inputSize), gorgonia.WithName("x"))
    	y := gorgonia.NewMatrix(g, tensor.Float32, gorgonia.WithShape(batchSize, 1), gorgonia.WithName("y"))
    
    	yhat := gorgonia.Must(gorgonia.Mul(x, w))
    	yhatSig := gorgonia.Must(gorgonia.Sigmoid(yhat)) // Apply sigmoid
    
    	// Binary cross-entropy loss
    	losses := gorgonia.Must(gorgonia.Sub(yhatSig, y))
    	squaredLoss := gorgonia.Must(gorgonia.Square(losses))
    	loss := gorgonia.Must(gorgonia.Mean(squaredLoss))
    
    	// Create gradients
    	_, err = gorgonia.Grad(loss, w)
    	if err != nil {
    		log.Fatalf("Failed to create gradients: %v", err)
    	}
    
    	// Initialize optimizer
    	optimizer := gorgonia.NewAdamSolver(gorgonia.WithLearnRate(0.001))
    
    	// Create the machine
    	machine := gorgonia.NewTapeMachine(g, gorgonia.BindDualValues(w))
    	defer machine.Close()
    
    	for epoch := 0; epoch < numEpochs; epoch++ {
    		var epochLoss float64
    		for start := 0; start < totalSamples; start += batchSize {
    			end := start + batchSize
    			if end > totalSamples {
    				end = totalSamples
    			}
    
    			batchSizeActual := end - start
    			xValData := make([]float32, batchSizeActual*inputSize)
    			yValData := make([]float32, batchSizeActual)
    
    			for i := start; i < end; i++ {
    				tokens := Tokenize(parsedData[i].Input)
    				inputVec := Vectorize(tokens, vocab)
    				copy(xValData[(i-start)*inputSize:(i-start+1)*inputSize], inputVec)
    
    				// Parse Output as float32
    				if parsedData[i].Output == "1" {
    					yValData[i-start] = 1.0
    				} else {
    					yValData[i-start] = 0.0
    				}
    			}
    
    			xVal := tensor.New(tensor.WithShape(batchSizeActual, inputSize), tensor.Of(tensor.Float32), tensor.WithBacking(xValData))
    			yVal := tensor.New(tensor.WithShape(batchSizeActual, 1), tensor.Of(tensor.Float32), tensor.WithBacking(yValData))
    
    			// Set node values using gorgonia.Let
    			if err := gorgonia.Let(x, xVal); err != nil {
    				log.Fatalf("Error setting x value: %v", err)
    			}
    			if err := gorgonia.Let(y, yVal); err != nil {
    				log.Fatalf("Error setting y value: %v", err)
    			}
    
    			if err := machine.RunAll(); err != nil {
    				log.Fatalf("Machine run error during training: %v", err)
    			}
    
    			// Get loss value
    			lossData := loss.Value().Data()
    			lossVal, ok := lossData.(float32)
    			if !ok {
    				log.Fatalf("Unexpected type for loss value: %T", lossData)
    			}
    
    			epochLoss += float64(lossVal) * float64(batchSizeActual)
    
    			// Update weights
    			nodes := gorgonia.Nodes{w}
    			err = optimizer.Step(gorgonia.NodesToValueGrads(nodes))
    			if err != nil {
    				log.Fatalf("Error during optimizer step: %v", err)
    			}
    
    			machine.Reset()
    		}
    
    		averageLoss := epochLoss / float64(totalSamples)
    		fmt.Printf("Epoch %d/%d - Loss: %.4f\n", epoch+1, numEpochs, averageLoss)
    	}
    
    	fmt.Println("Model training complete")
    
    	// Save weights to models directory
    	if err := SaveWeights(w.Value().(*tensor.Dense), "models/model_weights.gob"); err != nil {
    		log.Fatalf("Error saving model weights: %v", err)
    	} else {
    		fmt.Println("Model weights saved successfully.")
    	}
    }
    
    // interactWithModel allows interaction with the trained model
    func interactWithModel() {
    	weightsTensor, err := LoadWeights("models/model_weights.gob")
    	if err != nil {
    		log.Fatalf("Error loading model weights: %v", err)
    	}
    
    	vocab, err := LoadVocabulary("models/vocab.gob")
    	if err != nil {
    		log.Fatalf("Error loading vocabulary: %v", err)
    	}
    
    	inputSize := len(vocab)
    	reader := bufio.NewReader(os.Stdin)
    
    	// Load training data to retrieve responses
    	parsedData, err := LoadFiles("train", maxTrainMessages)
    	if err != nil {
    		log.Fatalf("Error loading training data for responses: %v", err)
    	}
    
    	// Separate responses based on labels
    	positiveResponses := []string{}
    	negativeResponses := []string{}
    
    	for _, item := range parsedData {
    		if item.Output == "1" {
    			positiveResponses = append(positiveResponses, item.Output)
    		} else {
    			negativeResponses = append(negativeResponses, item.Output)
    		}
    	}
    
    	// Seed the random number generator
    	rand.Seed(time.Now().UnixNano())
    
    	// Create graph for prediction
    	g := gorgonia.NewGraph()
    	w := gorgonia.NewMatrix(g, tensor.Float32, gorgonia.WithShape(inputSize, 1), gorgonia.WithValue(weightsTensor))
    
    	x := gorgonia.NewMatrix(g, tensor.Float32, gorgonia.WithShape(1, inputSize))
    	yhat := gorgonia.Must(gorgonia.Mul(x, w))
    	yhatSig := gorgonia.Must(gorgonia.Sigmoid(yhat)) // Apply sigmoid
    
    	machine := gorgonia.NewTapeMachine(g, gorgonia.BindDualValues(w))
    	defer machine.Close()
    
    	fmt.Println("Start interacting with the model. Type 'exit' to quit.")
    
    	for {
    		fmt.Print("You: ")
    		text, _ := reader.ReadString('\n')
    		text = strings.TrimSpace(text)
    
    		if strings.ToLower(text) == "exit" {
    			fmt.Println("Goodbye!")
    			break
    		}
    
    		tokens := Tokenize(text)
    		inputVec := Vectorize(tokens, vocab)
    
    		xValData := make([]float32, inputSize)
    		copy(xValData, inputVec)
    		xVal := tensor.New(tensor.WithShape(1, inputSize), tensor.Of(tensor.Float32), tensor.WithBacking(xValData))
    
    		// Set the value of x
    		if err := gorgonia.Let(x, xVal); err != nil {
    			log.Fatalf("Error setting x value: %v", err)
    		}
    
    		if err := machine.RunAll(); err != nil {
    			log.Fatalf("Machine run error during interaction: %v", err)
    		}
    
    		predictionsVal := yhatSig.Value().Data().([]float32)
    		predictedProbability := predictionsVal[0]
    		predictedOutput := 0
    		if predictedProbability >= 0.5 {
    			predictedOutput = 1
    		}
    
    		var response string
    		if predictedOutput == 1 {
    			if len(positiveResponses) > 0 {
    				response = positiveResponses[rand.Intn(len(positiveResponses))]
    			} else {
    				response = "Positive Response" // Fallback if no responses are available
    			}
    		} else {
    			if len(negativeResponses) > 0 {
    				response = negativeResponses[rand.Intn(len(negativeResponses))]
    			} else {
    				response = "Negative Response" // Fallback if no responses are available
    			}
    		}
    
    		fmt.Printf("AI: %s (Confidence: %.2f%%)\n", response, predictedProbability*100)
    		machine.Reset()
    	}
    }
    
    // validateModel performs validation of the model using data from validationDir
    func validateModel(validationDir string) {
    	weightsTensor, err := LoadWeights("models/model_weights.gob")
    	if err != nil {
    		log.Fatalf("Error loading model weights: %v", err)
    	}
    
    	vocab, err := LoadVocabulary("models/vocab.gob")
    	if err != nil {
    		log.Fatalf("Error loading vocabulary: %v", err)
    	}
    
    	validationData, err := LoadFiles(validationDir, maxTrainMessages) // Use appropriate limit
    	if err != nil {
    		log.Fatalf("Error loading validation files: %v", err)
    	}
    
    	fmt.Printf("Loaded %d validation records\n", len(validationData))
    
    	g := gorgonia.NewGraph()
    	w := gorgonia.NewMatrix(g, tensor.Float32, gorgonia.WithShape(len(vocab), 1), gorgonia.WithValue(weightsTensor))
    
    	x := gorgonia.NewMatrix(g, tensor.Float32, gorgonia.WithShape(1, len(vocab)))
    	y := gorgonia.NewMatrix(g, tensor.Float32, gorgonia.WithShape(1, 1))
    
    	yhat := gorgonia.Must(gorgonia.Mul(x, w))
    	yhatSig := gorgonia.Must(gorgonia.Sigmoid(yhat)) // Apply sigmoid
    
    	machine := gorgonia.NewTapeMachine(g, gorgonia.BindDualValues(w))
    	defer machine.Close()
    
    	var totalLoss float64
    	correct := 0
    	total := len(validationData)
    
    	for idx, item := range validationData {
    		inputTokens := Tokenize(item.Input)
    		inputVec := Vectorize(inputTokens, vocab)
    
    		xValData := make([]float32, len(vocab))
    		copy(xValData, inputVec)
    		xVal := tensor.New(tensor.WithShape(1, len(vocab)), tensor.Of(tensor.Float32), tensor.WithBacking(xValData))
    
    		// Parse Output as float32
    		var expectedOutput float32
    		if item.Output == "1" {
    			expectedOutput = 1.0
    		} else {
    			expectedOutput = 0.0
    		}
    
    		yValData := []float32{expectedOutput}
    		yVal := tensor.New(tensor.WithShape(1, 1), tensor.Of(tensor.Float32), tensor.WithBacking(yValData))
    
    		// Set the values of x and y
    		if err := gorgonia.Let(x, xVal); err != nil {
    			log.Fatalf("Error setting x value: %v", err)
    		}
    		if err := gorgonia.Let(y, yVal); err != nil {
    			log.Fatalf("Error setting y value: %v", err)
    		}
    
    		if err := machine.RunAll(); err != nil {
    			log.Fatalf("Machine run error during validation: %v", err)
    		}
    
    		predictionsVal := yhatSig.Value().Data().([]float32)
    		predictedProbability := predictionsVal[0]
    		predictedOutput := 0
    		if predictedProbability >= 0.5 {
    			predictedOutput = 1
    		}
    
    		// Calculate binary cross-entropy for the current example
    		epsilon := float32(1e-7) // To prevent log(0)
    		p := float32(math.Min(math.Max(float64(predictedProbability), float64(epsilon)), 1-1e-7))
    		lossVal := -expectedOutput*float32(math.Log(float64(p))) - (1-expectedOutput)*float32(math.Log(float64(1-p)))
    		totalLoss += float64(lossVal)
    
    		if predictedOutput == int(expectedOutput) {
    			correct++
    		}
    
    		machine.Reset()
    
    		// Optionally: display progress
    		if (idx+1)%1000 == 0 || idx+1 == total {
    			fmt.Printf("Validated %d/%d examples\n", idx+1, total)
    		}
    	}
    
    	averageLoss := totalLoss / float64(total)
    	accuracy := float64(correct) / float64(total) * 100
    	fmt.Printf("Validation Loss: %.4f\n", averageLoss)
    	fmt.Printf("Validation Accuracy: %.2f%%\n", accuracy)
    }
    
    // main is the entry point of the program
    func main() {
    	fmt.Print("Enter mode of operation (t for training, v for validation, d for dialogue): ")
    	scanner := bufio.NewScanner(os.Stdin)
    	scanner.Scan()
    	mode := scanner.Text()
    
    	switch mode {
    	case "t":
    		trainModel("train", maxTrainMessages)
    	case "v":
    		validateModel("valid")
    	case "d":
    		interactWithModel()
    	default:
    		fmt.Println("Unknown mode. Use 't' for training, 'v' for validation, or 'd' for dialogue.")
    	}
    }
    

    Explanation of Changes:

    1. Enhanced Response Generation in interactWithModel:

      • Loading Training Data for Responses: The function now loads the training data to access possible responses. It separates the responses into positive and negative categories based on the labels.
      • Random Selection of Responses: Instead of returning fixed responses, the program randomly selects a response from the appropriate category (positive or negative) based on the model's prediction.
      • Random Seed Initialization: To ensure that the responses are randomized each time, the random number generator is seeded with the current time.
      • Confidence Display: Along with the generated response, the AI now displays the confidence level (probability) of the prediction.
    2. Additional Error Handling and Fallbacks:

      • Fallback Responses: In cases where there are no available responses in the training data for a particular category, the program falls back to the default "Positive Response" or "Negative Response."
      • User-Friendly Interactions: The interaction loop now clearly indicates when the user can exit and how to do so.
    3. Maintained Existing Functionality:

      • All existing functions like LoadFiles, Tokenize, BuildVocab, Vectorize, SaveWeights, LoadWeights, SaveVocabulary, LoadVocabulary, sigmoid, trainModel, and validateModel remain unchanged to preserve the original functionality.
    4. Code Organization and Readability:

      • Proper comments and structured code segments are maintained for better readability and maintenance.

    How It Works:

    • Training Mode (t):

      • Loads training data from the train directory.
      • Builds a vocabulary and trains a simple logistic regression model using Gorgonia.
      • Saves the model weights and vocabulary for later use.
    • Validation Mode (v):

      • Loads the trained model and vocabulary.
      • Validates the model using data from the valid directory.
      • Reports the validation loss and accuracy.
    • Dialogue Mode (d):

      • Loads the trained model and vocabulary.
      • Loads the training data to prepare a pool of possible responses categorized as positive or negative.
      • Allows the user to input questions or statements.
      • The model predicts whether the input is positive or negative.
      • Based on the prediction, the program selects a random response from the corresponding category and displays it along with the confidence level.

    Usage:

    1. Training the Model:

      go run your_program.go
      
      • When prompted, enter t to start training.
      • Ensure that your training data is placed in the train directory.
    2. Validating the Model:

      go run your_program.go
      
      • When prompted, enter v to start validation.
      • Ensure that your validation data is placed in the valid directory.
    3. Interacting with the Model:

      go run your_program.go
      
      • When prompted, enter d to start the dialogue mode.
      • Type your inputs, and the AI will respond accordingly.
      • Type exit to quit the dialogue.

    Notes:

    • Data Format:

      • Ensure that your training and validation data are in .csv or .txt formats as expected by the LoadFiles function.
      • For .csv files, the first column should be the input and the second column the output label (1 for positive, 0 for negative).
      • For .txt files, each line should be in the format input:output.
    • Model Limitations:

      • The current model is a simple logistic regression classifier. While it can categorize inputs as positive or negative, generating truly dynamic and contextually relevant responses would require a more sophisticated model architecture, such as a sequence-to-sequence model or transformer-based model.
      • The implemented response generation is retrieval-based, selecting existing responses from the training data rather than generating new text.
    • Extensibility:

      • For more advanced response generation, consider integrating additional layers or switching to a different model architecture that supports text generation.
      • Incorporating natural language generation techniques can significantly enhance the quality and relevance of the AI's responses.

    This updated script should meet your requirements by providing more meaningful and varied responses based on the trained data while maintaining the existing functionality. Feel free to further modify and extend the program to better suit your needs!