Kode python analisis sentimen teks berita berbahasa indonesia berikut menggunakan metode…
- User
Kode python analisis sentimen teks berita berbahasa indonesia berikut menggunakan metode gabungan Bi-LSTM dan CNN kemudian dilanjutkan dengan SVM layer. Kode ini masih menggunakan vektor kata Embedding. Tolong modifikasi vektor kata tersebut menggunakan Word2Vec untuk Bahasa Indonesia! import numpy as np import pandas as pd import pickle import re from sklearn.model_selection import train_test_split from sklearn.preprocessing import LabelEncoder from sklearn.metrics import confusion_matrix, classification_report from sklearn.pipeline import Pipeline from sklearn.svm import SVC from keras.preprocessing.text import Tokenizer from keras.utils import pad_sequences from keras.models import Sequential from keras.layers import Dense, Embedding, SpatialDropout1D, Conv1D, MaxPooling1D, GlobalMaxPooling1D, Bidirectional, LSTM from keras.callbacks import EarlyStopping from sklearn.feature_extraction.text import TfidfVectorizer
Membaca dataset
data_list = pickle.load(open('filtered_pre_processed_berita_joined_done.pkl', 'rb')) data = pd.DataFrame(data_list, columns=['judul', 'isi', 'pre_processed', 'Label', 'Nama_Pelabel']) data['Isi Berita'] = data['pre_processed']
Tokenisasi data
tokenizer = Tokenizer(num_words=5000) tokenizer.fit_on_texts(data['Isi Berita']) X = tokenizer.texts_to_sequences(data['Isi Berita']) X = pad_sequences(X, maxlen=300)
Encoding label
le = LabelEncoder() y = le.fit_transform(data['Label'])
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)
model = Sequential() model.add(Embedding(input_dim=5000, output_dim=128, input_length=300)) model.add(SpatialDropout1D(0.4)) model.add(Bidirectional(LSTM(196, dropout=0.2, recurrent_dropout=0.2, return_sequences=True))) model.add(Conv1D(filters=32, kernel_size=3, activation='relu')) model.add(MaxPooling1D(pool_size=2)) model.add(Conv1D(filters=64, kernel_size=3, activation='relu')) model.add(MaxPooling1D(pool_size=2)) model.add(Conv1D(filters=128, kernel_size=3, activation='relu')) model.add(GlobalMaxPooling1D()) model.add(Dense(1, activation='sigmoid')) model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
history = model.fit(X_train, y_train, epochs=5, batch_size=256, validation_split=0.2, callbacks=[EarlyStopping(monitor='val_loss', patience=2)])
Menghapus layer Dense dengan aktivasi 'sigmoid' dan mengambil embedding dari X_train dan X_test
model.pop() train_embed = model.predict(X_train) test_embed = model.predict(X_test)
Membuat pipeline untuk menggabungkan TfidfVectorizer dan SVM classifier
svm_clf = Pipeline([ ('tfidf', TfidfVectorizer()), ('clf', SVC(kernel='linear', probability=True)) ])
Memformat ulang data ke format teks sederhana
X_train_text = [' '.join([str(num) for num in x]) for x in X_train] X_test_text = [' '.join([str(num) for num in x]) for x in X_test]
Melatih SVM classifier dengan data berformat teks
svm_clf.fit(X_train_text, y_train)
Evaluasi akurasi dan laporan klasifikasi dari SVM classifier
accuracy = svm_clf.score(X_test_text, y_test) y_pred = svm_clf.predict(X_test_text)
print("Accuracy: %.2f%%" % (accuracy*100)) print("Classification Report:\n\n", classification_report(y_test, y_pred))
- Assistant
Untuk mengganti vektor kata Embedding dengan Word2Vec, Anda perlu melatih model Word2Vec atau menggunakan model Word2Vec yang telah dilatih sebelumnya. Berikut adalah modifikasi kode yang menggunakan Word2Vec untuk bahasa Indonesia:
import numpy as np import pandas as pd import pickle import re from gensim.models import Word2Vec from sklearn.model_selection import train_test_split from sklearn.preprocessing import LabelEncoder from sklearn.metrics import confusion_matrix, classification_report from sklearn.pipeline import Pipeline from sklearn.svm import SVC from keras.preprocessing.text import Tokenizer from keras.utils import pad_sequences from keras.models import Sequential from keras.layers import Dense, Embedding, SpatialDropout1D, Conv1D, MaxPooling1D, GlobalMaxPooling1D, Bidirectional, LSTM from keras.callbacks import EarlyStopping from sklearn.feature_extraction.text import TfidfVectorizer # Membaca dataset data_list = pickle.load(open('filtered_pre_processed_berita_joined_done.pkl', 'rb')) data = pd.DataFrame(data_list, columns=['judul', 'isi', 'pre_processed', 'Label', 'Nama_Pelabel']) data['Isi Berita'] = data['pre_processed'] # Tokenisasi data texts = data['Isi Berita'].apply(str.split) embedding_dim = 128 max_len = 300 # Latih model Word2Vec atau gunakan model yang telah dilatih sebelumnya word_model = Word2Vec(texts, size=embedding_dim, window=5, min_count=1, workers=4) # Fungsi untuk mengubah teks menjadi vektor berdasarkan word2vec def text_to_vec(text): vec = np.zeros((max_len, embedding_dim)) text = text[:max_len] for i, token in enumerate(text): vec[i] = word_model.wv[token] return vec X = np.zeros((len(texts), max_len, embedding_dim)) for i, text in enumerate(texts): X[i] = text_to_vec(text) # Encoding label le = LabelEncoder() y = le.fit_transform(data['Label']) X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42) model = Sequential() model.add(LSTM(196, dropout=0.2, recurrent_dropout=0.2, return_sequences=True, input_shape=(max_len, embedding_dim))) model.add(Bidirectional(LSTM(196, dropout=0.2, recurrent_dropout=0.2, return_sequences=True))) model.add(Conv1D(filters=32, kernel_size=3, activation='relu')) model.add(MaxPooling1D(pool_size=2)) model.add(Conv1D(filters=64, kernel_size=3, activation='relu')) model.add(MaxPooling1D(pool_size=2)) model.add(Conv1D(filters=128, kernel_size=3, activation='relu')) model.add(GlobalMaxPooling1D()) model.add(Dense(1, activation='sigmoid')) model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy']) history = model.fit(X_train, y_train, epochs=5, batch_size=256, validation_split=0.2, callbacks=[EarlyStopping(monitor='val_loss', patience=2)]) # Menghapus layer Dense dengan aktivasi 'sigmoid' dan mengambil embedding dari X_train dan X_test model.pop() train_embed = model.predict(X_train) test_embed = model.predict(X_test) # Membuat pipeline untuk menggabungkan TfidfVectorizer dan SVM classifier svm_clf = Pipeline([ ('tfidf', TfidfVectorizer()), ('clf', SVC(kernel='linear', probability=True)) ]) # Memformat ulang data ke format teks sederhana X_train_text = [' '.join([str(num) for num in x]) for x in X_train.reshape((-1, max_len*embedding_dim))] X_test_text = [' '.join([str(num) for num in x]) for x in X_test.reshape((-1, max_len*embedding_dim))] # Melatih SVM classifier dengan data berformat teks svm_clf.fit(X_train_text, y_train) # Evaluasi akurasi dan laporan klasifikasi dari SVM classifier accuracy = svm_clf.score(X_test_text, y_test) y_pred = svm_clf.predict(X_test_text) print("Accuracy: %.2f%%" % (accuracy*100)) print("Classification Report:\n\n", classification_report(y_test, y_pred))Dalam kode ini, saya menggunakan model Word2Vec yang dilatih dari teks dataset. Selain itu, saya juga mengganti layer Embedding dengan layer LSTM biasa dengan input_shape yang sesuai. Anda bisa menggunakan model Word2Vec yang telah dilatih sebelumnya dengan menggantikan proses pelatihan model dengan hanya memuat model yang ada.