GENERATE TEXTBLOB

#Install Library

# !pip install tweet-preprocessor
# !pip install textblob
# !pip install wordcloud
# !pip  install Sastrawi
# !pip install nltk  
# !pip install tweet-preprocessor
# !pip install googletrans==4.0.0-rc1
# # c:\programdata\anaconda3\lib\site-packages (3.8.1)
#pip install scikit-learn

print('Start...')

#Import Library
import pandas as pd
import re
import seaborn as sns
import matplotlib.pyplot as plt
import Sastrawi
from Sastrawi.StopWordRemover.StopWordRemoverFactory import StopWordRemoverFactory, StopWordRemover, ArrayDictionary
from Sastrawi.Stemmer.StemmerFactory import StemmerFactory
import preprocessor as p
from textblob import TextBlob
import nltk
from nltk.stem import PorterStemmer
from nltk.tokenize import word_tokenize
from googletrans import Translator
 
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.model_selection import train_test_split
from sklearn.naive_bayes import MultinomialNB 
from sklearn.tree import DecisionTreeClassifier
from sklearn.neighbors import KNeighborsClassifier
from sklearn.metrics import accuracy_score, classification_report, confusion_matrix, ConfusionMatrixDisplay
import matplotlib.pyplot as plt
import seaborn as sns
nltk.download('punkt')

import re
import pandas as pd
from Sastrawi.Stemmer.StemmerFactory import StemmerFactory
from Sastrawi.StopWordRemover.StopWordRemoverFactory import StopWordRemoverFactory, StopWordRemover, ArrayDictionary

# Definisi kamus normalisasi
norm = {
    ' bgt ': ' banget ', ' gak ': ' tidak ', ' bs ': ' bisa ', ' emg ': ' memang ',
    ' yg ': ' yang ', ' g ': ' tidak', ' udah': ' sudah ', ' gini ': ' begini ',
    ' d ': ' di ', ' ad ': ' ada ', ' bhw ': ' bahwa ', ' tp ': ' tetapi ',
    ' sy ': ' saya ', ' ga ': ' tidak ', ' bkt ': ' bukti ', ' jt ': ' juta ',
    ' ajah ': ' saja ', ' gw ': ' saya '
}

# Fungsi untuk normalisasi, stopword removal, dan stemming dalam satu fungsi
def process_text(text):
    # Pastikan input adalah string
    if not isinstance(text, str):
        text = str(text)
    
    # Normalisasi dan pembersihan teks
    text = re.sub(r'@[A-Za-z0-9_]+', '', text)
    text = re.sub(r'#\w+', '', text)
    text = re.sub(r'RT[\s]+', '', text)
    text = re.sub(r'https?://\S+', '', text)
    text = re.sub(r'[^A-Za-z0-9 ]', '', text)
    text = re.sub(r'\s+', ' ', text).strip().lower()

    # Normalisasi menggunakan kamus
    for key, value in norm.items():
        text = text.replace(key, value)

    # Menghapus stopwords yang didefinisikan
    stop_words = stop_words_remover_new.remove(text)

    # Stemming
    stemmed_text = stemmer.stem(stop_words)

    return stemmed_text

# Fungsi clean_string tetap digunakan di akhir proses untuk pembersihan tambahan
def clean_string(text):
    text = text.strip()
    text = re.sub(r'\s+', ' ', text)
    
    # Menghilangkan kata-kata tertentu
    words_to_remove = ["ahok", "hari", "anies","dgn","dengan","atau","org","orang","utk"," nya","pak","atau","djarot","aku","saya","jkw","prabowo","jkt","jakarta","pks","pdip","sama"]
    pattern = re.compile(r'\b(' + '|'.join(words_to_remove) + r')\b', re.IGNORECASE)
    text = pattern.sub('', text)
    
    # Menghilangkan angka dalam bentuk kata dan angka
    numbers_in_words = ["satu", "dua", "tiga", "empat", "lima", "enam", "tujuh", "delapan", "sembilan", "nol","ribu","juta"]
    pattern = re.compile(r'\b(' + '|'.join(numbers_in_words) + r')\b', re.IGNORECASE)
    text = pattern.sub('', text)
    text = re.sub(r'\b\d+\b', '', text)
    
    # Menghilangkan kata yang kurang dari 3 huruf
    text = ' '.join([word for word in text.split() if len(word) >= 3])
    
    # Menghilangkan spasi yang lebih dari satu lagi setelah penghapusan kata-kata tertentu dan angka
    text =re.sub(r'\d+', '', text)
    text = re.sub(r'\s+', ' ', text).strip()
    return text 

# Inisialisasi StopWordRemover
more_stop_words =["tidak","buk","tidak","tidak"]
stop_words = StopWordRemoverFactory().get_stop_words()
stop_words.extend(more_stop_words)
new_array = ArrayDictionary(stop_words)
stop_words_remover_new = StopWordRemover(new_array)

# Inisialisasi Stemmer
factory = StemmerFactory()
stemmer = factory.create_stemmer()

#============================================================
# Membaca dataset
df = pd.read_csv("cthdata.csv")
df = df[['text']]
print(df.head())
print(df.shape)

# Mengatasi nilai NaN atau nilai non-string
df['text'] = df['text'].fillna('').astype(str)

# Proses teks secara keseluruhan
df['text'] = df['text'].apply(lambda x: clean_string(process_text(x)))

# Simpan hasil
df['text'].to_csv("DataStemFilter.csv", index=False)
print('Sukses normalisasi data...')
import time        
from googletrans import Translator
from textblob import TextBlob
import pandas as pd

def translate_with_retry(text, src='id', dest='en', max_retries=3):
    for attempt in range(max_retries):
        try:
            return translator.translate(text, src=src, dest=dest).text
        except Exception as e:
            print(f"Attempt {attempt + 1} failed: {e}")
            time.sleep(1)  # Tunggu 1 detik sebelum mencoba lagi
    return None

def translate_simple(tweet):
    translator = Translator()
    try:
        translated_text = translator.translate(tweet, src='id', dest='en').text
        return translated_text
    except Exception as e:
        print(f"Terjadi kesalahan dalam proses terjemahan: {e}")
        return None

# Load Data yang sudah ternormalisasi (StopWord+Lemmatizer+Stemming)
data = pd.read_csv('DataStemFilter.csv')
data_tweet = list(data['text'])
polaritas = 0
status = []
full_text2 = []
polarity = []
total_positif = total_negatif = total_netral = total = 0

for i, tweet in enumerate(data_tweet):
    # Periksa apakah elemen adalah string, jika tidak konversi ke string
    if isinstance(tweet, float):
        tweet = str(tweet) if not pd.isna(tweet) else ""
    elif not isinstance(tweet, str):
        tweet = str(tweet)
    
    # Analisis hanya jika tweet bukan string kosong
    if tweet:
        translated_text = translate_simple(tweet)
        #translated_text = translator.translate(tweet, src='id', dest='en').text 
        #translated_text = translator.translate(tweet, src='id', dest='en', timeout=30).text  # Timeout 10 detik
        #translated_text = translate_with_retry(tweet) 
        
        # Jika terjemahan berhasil, tambahkan ke daftar, jika tidak tambahkan None
        if translated_text:
            full_text2.append(translated_text)
            # Membuat objek TextBlob dari teks yang diterjemahkan
            analysis = TextBlob(str(translated_text))  # tweet
            polar = analysis.polarity
            polaritas += polar
            polarity.append(polar)
            if analysis.sentiment.polarity > 0.0:
                total_positif += 1
                status.append('Positif')
            elif analysis.sentiment.polarity == 0.0:
                total_netral += 1
                status.append('Netral')
            else:
                total_negatif += 1
                status.append('Negatif')
        else:
            # Jika terjemahan gagal, tambahkan nilai default
            full_text2.append(None)
            polarity.append(0)
            status.append('Tidak Ada Data')
    else:
        # Jika tweet kosong, tambahkan nilai default
        full_text2.append(None)
        polarity.append(0)
        status.append('Tidak Ada Data')
    
    total += 1

# Pastikan panjang semua daftar sama dengan panjang DataFrame
if len(full_text2) < len(data):
    missing_entries = len(data) - len(full_text2)
    full_text2.extend([None] * missing_entries)
    polarity.extend([0] * missing_entries)
    status.extend(['Tidak Ada Data'] * missing_entries)

# Menambahkan kolom 'klasifikasi' ke DataFrame
data['translate'] = full_text2
data['polarity'] = polarity
data['klasifikasi'] = status

# Output hasil analisis
print(f'Hasil Analisis Data:\nPositif = {total_positif}\nNetral = {total_netral}\nNegatif = {total_negatif}')
print(f'\nTotal Data : {total}')
data.to_csv("DataAutoKlasifikasi.csv", index=False)
print('Sukses update dataset...')


import seaborn as sns
sns.set_theme()
labels = ['Positif', 'Netral', 'Negatif']
counts = [total_positif, total_netral, total_negatif]

def show_bar(labels, counts):
    sns.set_theme()
    plt.figure(figsize=(8, 6))
    sns.barplot(x=labels, y=counts)
    plt.title('Distribusi Sentimen')
    plt.xlabel('Kategori Sentimen')
    plt.ylabel('Jumlah')
    plt.show()

# Memanggil fungsi untuk menampilkan bar plot
show_bar(labels, counts)


import pandas as pd
print('Isi dataset yang terbaru....')
dataNorm = pd.read_csv('DataAutoKlasifikasi.csv')
print(dataNorm)