diff --git a/deployment/api/preprocessing.py b/deployment/api/preprocessing.py index 7f822e8..1bed0c2 100644 --- a/deployment/api/preprocessing.py +++ b/deployment/api/preprocessing.py @@ -5,7 +5,7 @@ from nltk.corpus import stopwords from nltk.tokenize import word_tokenize from nltk.stem import WordNetLemmatizer -# Assurer que les ressources nécessaires sont disponibles +# Ensure that the necessary resources are available try: stopwords.words('english') word_tokenize("test") @@ -17,28 +17,50 @@ except LookupError: nltk.download('wordnet') nltk.download('omw-1.4') +# PREPROCESSING FUNCTION FOR CLASSIC AND ADVANCED MODELS def preprocess_tweet(tweet): """ - Tweet pre-processing function for the Sentiment Analysis API + Preprocesses a tweet for classic ML models or word embedding models (e.g., Word2Vec, GloVe). + The text is cleaned, normalized, lemmatized, and stripped of unnecessary noise. """ + + # Ensure input is a string if not isinstance(tweet, str): return "" - tweet = html.unescape(tweet) + + # Convert text to lowercase tweet = tweet.lower() + + # Replace URLs with a generic token tweet = re.sub(r'https?://\S+|www\.\S+', '', tweet) + + # Replace mentions (@username) with a generic token tweet = re.sub(r'@\w+', '', tweet) + + # Separate hashtags from the word to preserve semantics during tokenization tweet = re.sub(r'#(\w+)', r'# \1', tweet) + + # Decode HTML entities (e.g., & → &) + tweet = html.unescape(tweet) + + # Remove unwanted punctuation and special characters (except useful ones like < > @ # ! ?) tweet = re.sub(r'[^\w\s<>@#!?]', '', tweet) - + + # Remove non-printable/control characters (e.g., \n, \t, emojis, corrupted unicode) + tweet = re.sub(r'[^\x20-\x7E]', '', tweet) + + # Tokenize the tweet into individual words tokens = word_tokenize(tweet) - + + # Lemmatize each token to its base form (e.g., "running" → "run") lemmatizer = WordNetLemmatizer() tokens = [lemmatizer.lemmatize(token) for token in tokens] - + + # Remove stopwords, but keep negation-related words (important for sentiment) stop_words = set(stopwords.words('english')) important_words = {'no', 'not', 'nor', 'neither', 'never', 'nobody', 'none', 'nothing', 'nowhere'} stop_words = stop_words - important_words - tokens = [token for token in tokens if token not in stop_words] - + + # Reconstruct the cleaned tweet return ' '.join(tokens) \ No newline at end of file diff --git a/functions.py b/functions.py index 7663a65..4346fe4 100644 --- a/functions.py +++ b/functions.py @@ -4,68 +4,122 @@ import numpy as np import pandas as pd from multiprocessing import Pool -# NLTK +# NLTK imports for NLP preprocessing import nltk from nltk.corpus import stopwords from nltk.tokenize import word_tokenize from nltk.stem import WordNetLemmatizer -# Assure que les ressources nécessaires sont disponibles +# Ensure necessary NLTK resources are downloaded and available try: stopwords.words('english') word_tokenize("test") WordNetLemmatizer().lemmatize("testing") except LookupError: - nltk.download('punkt') - nltk.download('punkt_tab') - nltk.download('stopwords') - nltk.download('wordnet') - nltk.download('omw-1.4') # Pour les lemmes WordNet en anglais + nltk.download('punkt') # Tokenizer models + nltk.download('punkt_tab') # Additional tokenizer support + nltk.download('stopwords') # Stopwords lists + nltk.download('wordnet') # WordNet lexicon for lemmatization + nltk.download('omw-1.4') # WordNet lemmas data -# Convertir les labels : 0 pour négatif, 1 pour tout le reste +# Convert sentiment labels to binary: 0 for negative, 1 for all other classes def convert_sentiment_label(df): + """ + Convert the 'target' column in the DataFrame to binary sentiment labels: + - 0 for negative sentiment (label 0) + - 1 for any other sentiment label + Returns a new DataFrame with updated labels. + """ converted_target_data = df.copy() converted_target_data['target'] = converted_target_data['target'].apply(lambda x: 0 if x == 0 else 1) return converted_target_data -# Réduction de la taille du dataset pour équilibrer +# Downsample the dataset to balance classes by limiting samples per class def downsample_data(df, n_samples=50000, random_state=42): + """ + Perform downsampling to balance dataset classes. + Selects `n_samples` examples from both negative (0) and positive (1) classes. + Returns a balanced DataFrame with equal number of samples per class. + """ negative_samples = df[df['target'] == 0].sample(n=n_samples, random_state=random_state) positive_samples = df[df['target'] == 1].sample(n=n_samples, random_state=random_state) downsampled_data = pd.concat([negative_samples, positive_samples]) return downsampled_data -# Fonction principale de prétraitement +# PREPROCESSING FUNCTION FOR CLASSIC AND ADVANCED MODELS def preprocess_tweet(tweet): + """ + Preprocess a tweet for classic ML or word embedding models (e.g., Word2Vec, GloVe). + Steps: + - Normalize text (lowercase) + - Replace URLs and mentions with tokens + - Decode HTML entities + - Remove unwanted special characters and non-printable chars + - Tokenize text + - Lemmatize tokens + - Remove stopwords except negation words + Returns cleaned, normalized text as a single string. + """ + + # Ensure input is a string if not isinstance(tweet, str): return "" - tweet = html.unescape(tweet) + + # Convert to lowercase tweet = tweet.lower() + + # Replace URLs with placeholder token tweet = re.sub(r'https?://\S+|www\.\S+', '', tweet) + + # Replace user mentions with placeholder token tweet = re.sub(r'@\w+', '', tweet) + + # Separate hashtags from words to facilitate tokenization tweet = re.sub(r'#(\w+)', r'# \1', tweet) + + # Decode HTML entities (e.g., & to &) + tweet = html.unescape(tweet) + + # Remove unwanted punctuation and special characters except some useful ones tweet = re.sub(r'[^\w\s<>@#!?]', '', tweet) + # Remove non-printable and control characters (e.g., tabs, newlines, emojis) + tweet = re.sub(r'[^\x20-\x7E]', '', tweet) + + # Tokenize tweet into words tokens = word_tokenize(tweet) + # Lemmatize tokens to their base form lemmatizer = WordNetLemmatizer() tokens = [lemmatizer.lemmatize(token) for token in tokens] + # Remove stopwords but keep negation-related words to preserve sentiment meaning stop_words = set(stopwords.words('english')) important_words = {'no', 'not', 'nor', 'neither', 'never', 'nobody', 'none', 'nothing', 'nowhere'} stop_words = stop_words - important_words tokens = [token for token in tokens if token not in stop_words] + # Rebuild cleaned tweet text return ' '.join(tokens) -# Traitement en parallèle sur un DataFrame +# Parallel processing of a DataFrame using multiple CPU cores def process_in_parallel(df, func, n_jobs=4): + """ + Split a DataFrame into `n_jobs` parts and apply `func` to each part in parallel, + using multiprocessing Pool for speed-up. + Returns the concatenated processed DataFrame. + """ df_split = np.array_split(df, n_jobs) with Pool(n_jobs) as pool: df = pd.concat(pool.map(func, df_split)) return df -# Application du prétraitement sur une partition +# Apply preprocessing function on a DataFrame partition def apply_preprocessing(df_part): + """ + Apply tweet preprocessing to the 'text' column of a DataFrame partition. + Stores results in a new column 'processed_text'. + Returns the updated DataFrame partition. + """ df_part['processed_text'] = df_part['text'].apply(preprocess_tweet) return df_part diff --git a/presentation/récapitulatif modèle.xlsx b/presentation/récapitulatif modèle.xlsx new file mode 100644 index 0000000..00a62c7 Binary files /dev/null and b/presentation/récapitulatif modèle.xlsx differ