diff --git a/deployment/api/preprocessing.py b/deployment/api/preprocessing.py index 3a86053..39031e6 100644 --- a/deployment/api/preprocessing.py +++ b/deployment/api/preprocessing.py @@ -30,37 +30,51 @@ def preprocess_tweet(tweet): # Decode HTML entities (e.g., & → &) tweet = html.unescape(tweet) + # print(tweet + '\n') # Convert text to lowercase tweet = tweet.lower() + # print(tweet + '\n') # Replace URLs with a generic token tweet = re.sub(r'https?://\S+|www\.\S+', '', tweet) + # print(tweet + '\n') # Replace mentions (@username) with a generic token tweet = re.sub(r'@\w+', '', tweet) + # print(tweet + '\n') # Separate hashtags from the word to preserve semantics during tokenization tweet = re.sub(r'#(\w+)', r'# \1', tweet) + # print(tweet + '\n') # Remove unwanted punctuation and special characters (except useful ones like < > @ # ! ?) tweet = re.sub(r'[^\w\s<>@#!?]', '', tweet) + # print(tweet + '\n') # Remove non-printable/control characters (e.g., \n, \t, emojis, corrupted unicode) tweet = re.sub(r'[^\x20-\x7E]', '', tweet) + # print(tweet + '\n') # Tokenize the tweet into individual words tokens = word_tokenize(tweet) + # print(tokens) + # print('\n') # Lemmatize each token to its base form (e.g., "running" → "run") lemmatizer = WordNetLemmatizer() tokens = [lemmatizer.lemmatize(token) for token in tokens] + # print(tokens) + # print('\n') # Remove stopwords, but keep negation-related words (important for sentiment) stop_words = set(stopwords.words('english')) important_words = {'no', 'not', 'nor', 'neither', 'never', 'nobody', 'none', 'nothing', 'nowhere'} stop_words = stop_words - important_words tokens = [token for token in tokens if token not in stop_words] + # print(tokens) + # print('\n') + # Reconstruct the cleaned tweet return ' '.join(tokens) \ No newline at end of file diff --git a/deployment/api/tests/test_preprocessing.py b/deployment/api/tests/test_preprocessing.py index 56143e8..4838ccc 100644 --- a/deployment/api/tests/test_preprocessing.py +++ b/deployment/api/tests/test_preprocessing.py @@ -8,10 +8,10 @@ from preprocessing import preprocess_tweet def test_preprocess_tweet_removes_hashtags(): tweet = """ - Just flew w/ @AirParadis 🚀 & I’m in LOVE! Epic service, comfy seats. + Just flew w/ @AirParadis 🚀 & I’m in LOVE! Epic service, comfy seats, good prices at https://airparadis.com #bestflight #AirParadis """ - assert preprocess_tweet(tweet) == "flew w < MENTION > im love ! epic service comfy seat # bestflight # airparadis" + assert preprocess_tweet(tweet) == 'flew w < MENTION > im love ! epic service comfy seat good price < URL > # bestflight # airparadis' def test_preprocess_tweet_removes_mentions(): tweet = """ @@ -25,4 +25,4 @@ def test_preprocess_tweet_lowercases_text(): My 3rd time w/ @AirParadis & still amazed 😍 Crew = lovely à every step! #flyhappy #AirParadis """ - assert preprocess_tweet(tweet) == '3rd time w < MENTION > still amazed crew lovely every step ! # flyhappy # airparadis' \ No newline at end of file + assert preprocess_tweet(tweet) == '3rd time w < MENTION > still amazed crew lovely every step ! # flyhappy # airparadis' diff --git a/presentation/presentation_modeles.md b/presentation/presentation_modeles.md new file mode 100644 index 0000000..e69de29 diff --git a/presentation/presentation_modeles.xlsx b/presentation/presentation_modeles.xlsx new file mode 100644 index 0000000..040d42c Binary files /dev/null and b/presentation/presentation_modeles.xlsx differ diff --git a/presentation/récapitulatif modèle.xlsx b/presentation/récapitulatif modèle.xlsx deleted file mode 100644 index 00a62c7..0000000 Binary files a/presentation/récapitulatif modèle.xlsx and /dev/null differ diff --git a/presentation/tableau_comparatif_air_paradis_word2vec_glove.xlsx b/presentation/tableau_comparatif_air_paradis_word2vec_glove.xlsx new file mode 100644 index 0000000..0cd382a Binary files /dev/null and b/presentation/tableau_comparatif_air_paradis_word2vec_glove.xlsx differ diff --git a/presentation/~$presentation_modeles.xlsx b/presentation/~$presentation_modeles.xlsx new file mode 100644 index 0000000..1a50c8d Binary files /dev/null and b/presentation/~$presentation_modeles.xlsx differ