diff --git a/deployment/api/preprocessing.py b/deployment/api/preprocessing.py index 31bb6b7..3a86053 100644 --- a/deployment/api/preprocessing.py +++ b/deployment/api/preprocessing.py @@ -28,6 +28,9 @@ def preprocess_tweet(tweet): if not isinstance(tweet, str): return "" + # Decode HTML entities (e.g., & → &) + tweet = html.unescape(tweet) + # Convert text to lowercase tweet = tweet.lower() @@ -40,14 +43,11 @@ def preprocess_tweet(tweet): # Separate hashtags from the word to preserve semantics during tokenization tweet = re.sub(r'#(\w+)', r'# \1', tweet) - # Decode HTML entities (e.g., & → &) - tweet = html.unescape(tweet) - # Remove unwanted punctuation and special characters (except useful ones like < > @ # ! ?) tweet = re.sub(r'[^\w\s<>@#!?]', '', tweet) # Remove non-printable/control characters (e.g., \n, \t, emojis, corrupted unicode) - # tweet = re.sub(r'[^\x20-\x7E]', '', tweet) + tweet = re.sub(r'[^\x20-\x7E]', '', tweet) # Tokenize the tweet into individual words tokens = word_tokenize(tweet) diff --git a/deployment/api/tests/test_preprocessing.py b/deployment/api/tests/test_preprocessing.py index b5e9b60..0013fa4 100644 --- a/deployment/api/tests/test_preprocessing.py +++ b/deployment/api/tests/test_preprocessing.py @@ -8,10 +8,10 @@ from preprocessing import preprocess_tweet def test_preprocess_tweet_removes_hashtags(): tweet = """ - Just flew w/ @AirParadis 🚀 & I’m in LOVE! Épic service, comfy seats. + Just flew w/ @AirParadis 🚀 & I’m in LOVE! Epic service, comfy seats. #bestflight #AirParadis """ - assert preprocess_tweet(tweet) == "flew w < MENTION > im love ! épic service comfy seat # bestflight # airparadis" + assert preprocess_tweet(tweet) == "flew w < MENTION > im love ! epic service comfy seat # bestflight # airparadis" def test_preprocess_tweet_removes_mentions(): tweet = """ @@ -25,4 +25,12 @@ def test_preprocess_tweet_lowercases_text(): My 3rd time w/ @AirParadis & still amazed 😍 Crew = lovely à every step! #flyhappy #AirParadis """ - assert preprocess_tweet(tweet) == '3rd time w < MENTION > still amazed crew lovely à every step ! # flyhappy # airparadis' \ No newline at end of file + assert preprocess_tweet(tweet) == '3rd time w < MENTION > still amazed crew lovely à every step ! # flyhappy # airparadis' + + + +tweet = """ +Just flew w/ @AirParadis 🚀 & I’m in LOVE! Epic service, comfy seats. +#bestflight #AirParadis +""" +preprocess_tweet(tweet) \ No newline at end of file