import nltk nltk.download('punkt') import numpy as np from nltk.stem.porter import…
- User
import nltk nltk.download('punkt') import numpy as np from nltk.stem.porter import PorterStemmer stemmer = PorterStemmer()
def tokenize(sentence): return nltk.word_tokenize(sentence)
def stem(word): return stemmer.stem(word.lower())
def bag_of_words(tokenized_sentence, all_words): """ sentence = ["hello", "how", "are", "you"] words = ["hi", "hello", "I", "you", "bye", "thank", "cool"] bag = [0, 1, 0, 1, 0, 0, 0] """
tokenized_sentence = [stem(w) for w in tokenized_sentence] bag = np.zeros(len(all_words), dtype=np.float32) for idx, w in enumerate(all_words): if w in tokenized_sentence: bag[idx] = 1.0 return bagshuffle our features and turn into np.array
random.shuffle(bag) training = np.array(bag)
create train and test lists
train_x = list(training[:,0]) train_y = list(training[:,1])
sentence = ["hello", "how", "are", "you"]
words = ["hi", "hello", "I", "you", "bye", "thank", "cool"]
bag = bag_of_words(sentence, words)
print(bag)
perbaiki kode di atas. saya ingin menampilkan train_x dan train_y. masalahnya pada kode di atas terdapat error: bag is not define
- Assistant
It seems like you are missing the part of the code where you define the "bag" variable, which should be a list containing pairs of tokenized sentences and their corresponding labels. I'm assuming that you have a list of sentences and their labels. Let's call them "sentences" and "tags" respectively. Below I provided a corrected version of the code.
import nltk nltk.download('punkt') import numpy as np import random from nltk.stem.porter import PorterStemmer stemmer = PorterStemmer() def tokenize(sentence): return nltk.word_tokenize(sentence) def stem(word): return stemmer.stem(word.lower()) def bag_of_words(tokenized_sentence, all_words): tokenized_sentence = [stem(w) for w in tokenized_sentence] bag = np.zeros(len(all_words), dtype=np.float32) for idx, w in enumerate(all_words): if w in tokenized_sentence: bag[idx] = 1.0 return bag # Example sentences and tags sentences = ["hello, how are you?", "I am doing fine", "Bye, see you later!"] tags = ["greeting", "response", "goodbye"] # Tokenize the sentences tokenized_sentences = [tokenize(sentence) for sentence in sentences] # Create a list of all unique words in the tokenized_sentences all_words = set(stem(word) for sentence in tokenized_sentences for word in sentence) all_words = sorted(list(all_words)) # Create a list of pairs containing the bag of words representation and its corresponding label bag = [] for idx, tokenized_sentence in enumerate(tokenized_sentences): bag.append((bag_of_words(tokenized_sentence, all_words), tags[idx])) # Shuffle the bag and convert it to a numpy array random.shuffle(bag) training = np.array(bag, dtype=object) # Create train lists train_x = list(training[:, 0]) train_y = list(training[:, 1]) print("train_x:", train_x) print("train_y:", train_y)Replace the example sentences and tags with your own data if needed.