Skip to content
TheChatArchiveThe Chat Archive
gpt-4-0314· OpenAIAi2 WildChat-4.8M (AI2)

Kode python analisis sentimen teks berita berbahasa indonesia berikut menggunakan metode…

  1. User

    Kode python analisis sentimen teks berita berbahasa indonesia berikut menggunakan metode gabungan Bi-LSTM dan CNN kemudian dilanjutkan dengan SVM layer. Kode ini masih menggunakan vektor kata Embedding. Tolong modifikasi vektor kata tersebut menggunakan Word2Vec untuk Bahasa Indonesia! import numpy as np import pandas as pd import pickle import re from sklearn.model_selection import train_test_split from sklearn.preprocessing import LabelEncoder from sklearn.metrics import confusion_matrix, classification_report from sklearn.pipeline import Pipeline from sklearn.svm import SVC from keras.preprocessing.text import Tokenizer from keras.utils import pad_sequences from keras.models import Sequential from keras.layers import Dense, Embedding, SpatialDropout1D, Conv1D, MaxPooling1D, GlobalMaxPooling1D, Bidirectional, LSTM from keras.callbacks import EarlyStopping from sklearn.feature_extraction.text import TfidfVectorizer

    Membaca dataset

    data_list = pickle.load(open('filtered_pre_processed_berita_joined_done.pkl', 'rb')) data = pd.DataFrame(data_list, columns=['judul', 'isi', 'pre_processed', 'Label', 'Nama_Pelabel']) data['Isi Berita'] = data['pre_processed']

    Tokenisasi data

    tokenizer = Tokenizer(num_words=5000) tokenizer.fit_on_texts(data['Isi Berita']) X = tokenizer.texts_to_sequences(data['Isi Berita']) X = pad_sequences(X, maxlen=300)

    Encoding label

    le = LabelEncoder() y = le.fit_transform(data['Label'])

    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)

    model = Sequential() model.add(Embedding(input_dim=5000, output_dim=128, input_length=300)) model.add(SpatialDropout1D(0.4)) model.add(Bidirectional(LSTM(196, dropout=0.2, recurrent_dropout=0.2, return_sequences=True))) model.add(Conv1D(filters=32, kernel_size=3, activation='relu')) model.add(MaxPooling1D(pool_size=2)) model.add(Conv1D(filters=64, kernel_size=3, activation='relu')) model.add(MaxPooling1D(pool_size=2)) model.add(Conv1D(filters=128, kernel_size=3, activation='relu')) model.add(GlobalMaxPooling1D()) model.add(Dense(1, activation='sigmoid')) model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])

    history = model.fit(X_train, y_train, epochs=5, batch_size=256, validation_split=0.2, callbacks=[EarlyStopping(monitor='val_loss', patience=2)])

    Menghapus layer Dense dengan aktivasi 'sigmoid' dan mengambil embedding dari X_train dan X_test

    model.pop() train_embed = model.predict(X_train) test_embed = model.predict(X_test)

    Membuat pipeline untuk menggabungkan TfidfVectorizer dan SVM classifier

    svm_clf = Pipeline([ ('tfidf', TfidfVectorizer()), ('clf', SVC(kernel='linear', probability=True)) ])

    Memformat ulang data ke format teks sederhana

    X_train_text = [' '.join([str(num) for num in x]) for x in X_train] X_test_text = [' '.join([str(num) for num in x]) for x in X_test]

    Melatih SVM classifier dengan data berformat teks

    svm_clf.fit(X_train_text, y_train)

    Evaluasi akurasi dan laporan klasifikasi dari SVM classifier

    accuracy = svm_clf.score(X_test_text, y_test) y_pred = svm_clf.predict(X_test_text)

    print("Accuracy: %.2f%%" % (accuracy*100)) print("Classification Report:\n\n", classification_report(y_test, y_pred))

  2. Assistant

    Untuk mengganti vektor kata Embedding dengan Word2Vec, Anda perlu melatih model Word2Vec atau menggunakan model Word2Vec yang telah dilatih sebelumnya. Berikut adalah modifikasi kode yang menggunakan Word2Vec untuk bahasa Indonesia:

    import numpy as np
    import pandas as pd
    import pickle
    import re
    from gensim.models import Word2Vec
    from sklearn.model_selection import train_test_split
    from sklearn.preprocessing import LabelEncoder
    from sklearn.metrics import confusion_matrix, classification_report
    from sklearn.pipeline import Pipeline
    from sklearn.svm import SVC
    from keras.preprocessing.text import Tokenizer
    from keras.utils import pad_sequences
    from keras.models import Sequential
    from keras.layers import Dense, Embedding, SpatialDropout1D, Conv1D, MaxPooling1D, GlobalMaxPooling1D, Bidirectional, LSTM
    from keras.callbacks import EarlyStopping
    from sklearn.feature_extraction.text import TfidfVectorizer
    
    # Membaca dataset
    data_list = pickle.load(open('filtered_pre_processed_berita_joined_done.pkl', 'rb'))
    data = pd.DataFrame(data_list, columns=['judul', 'isi', 'pre_processed', 'Label', 'Nama_Pelabel'])
    data['Isi Berita'] = data['pre_processed']
    
    # Tokenisasi data
    texts = data['Isi Berita'].apply(str.split)
    embedding_dim = 128
    max_len = 300
    
    # Latih model Word2Vec atau gunakan model yang telah dilatih sebelumnya
    word_model = Word2Vec(texts, size=embedding_dim, window=5, min_count=1, workers=4)
    
    # Fungsi untuk mengubah teks menjadi vektor berdasarkan word2vec
    def text_to_vec(text):
        vec = np.zeros((max_len, embedding_dim))
        text = text[:max_len]
        for i, token in enumerate(text):
            vec[i] = word_model.wv[token]
        return vec
    
    X = np.zeros((len(texts), max_len, embedding_dim))
    for i, text in enumerate(texts):
        X[i] = text_to_vec(text)
    
    # Encoding label
    le = LabelEncoder()
    y = le.fit_transform(data['Label'])
    
    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)
    
    model = Sequential()
    model.add(LSTM(196, dropout=0.2, recurrent_dropout=0.2, return_sequences=True, input_shape=(max_len, embedding_dim)))
    model.add(Bidirectional(LSTM(196, dropout=0.2, recurrent_dropout=0.2, return_sequences=True)))
    model.add(Conv1D(filters=32, kernel_size=3, activation='relu'))
    model.add(MaxPooling1D(pool_size=2))
    model.add(Conv1D(filters=64, kernel_size=3, activation='relu'))
    model.add(MaxPooling1D(pool_size=2))
    model.add(Conv1D(filters=128, kernel_size=3, activation='relu'))
    model.add(GlobalMaxPooling1D())
    model.add(Dense(1, activation='sigmoid'))
    model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
    
    history = model.fit(X_train, y_train, epochs=5, batch_size=256, validation_split=0.2, callbacks=[EarlyStopping(monitor='val_loss', patience=2)])
    
    # Menghapus layer Dense dengan aktivasi 'sigmoid' dan mengambil embedding dari X_train dan X_test
    model.pop()
    train_embed = model.predict(X_train)
    test_embed = model.predict(X_test)
    
    # Membuat pipeline untuk menggabungkan TfidfVectorizer dan SVM classifier
    svm_clf = Pipeline([
        ('tfidf', TfidfVectorizer()),
        ('clf', SVC(kernel='linear', probability=True))
    ])
    
    # Memformat ulang data ke format teks sederhana
    X_train_text = [' '.join([str(num) for num in x]) for x in X_train.reshape((-1, max_len*embedding_dim))]
    X_test_text = [' '.join([str(num) for num in x]) for x in X_test.reshape((-1, max_len*embedding_dim))]
    
    # Melatih SVM classifier dengan data berformat teks
    svm_clf.fit(X_train_text, y_train)
    
    # Evaluasi akurasi dan laporan klasifikasi dari SVM classifier
    accuracy = svm_clf.score(X_test_text, y_test)
    y_pred = svm_clf.predict(X_test_text)
    
    print("Accuracy: %.2f%%" % (accuracy*100))
    print("Classification Report:\n\n", classification_report(y_test, y_pred))
    

    Dalam kode ini, saya menggunakan model Word2Vec yang dilatih dari teks dataset. Selain itu, saya juga mengganti layer Embedding dengan layer LSTM biasa dengan input_shape yang sesuai. Anda bisa menggunakan model Word2Vec yang telah dilatih sebelumnya dengan menggantikan proses pelatihan model dengan hanya memuat model yang ada.