preprocessing + unit tests updated

deployment/api/preprocessing.py

@@ -30,37 +30,51 @@ def preprocess_tweet(tweet):
 
     # Decode HTML entities (e.g., & → &)
     tweet = html.unescape(tweet)
+    # print(tweet + '\n')
     
     # Convert text to lowercase
     tweet = tweet.lower()
+    # print(tweet + '\n')
 
     # Replace URLs with a generic token
     tweet = re.sub(r'https?://\S+|www\.\S+', '<URL>', tweet)
+    # print(tweet + '\n')
 
     # Replace mentions (@username) with a generic token
     tweet = re.sub(r'@\w+', '<MENTION>', tweet)
+    # print(tweet + '\n')
 
     # Separate hashtags from the word to preserve semantics during tokenization
     tweet = re.sub(r'#(\w+)', r'# \1', tweet)
+    # print(tweet + '\n')
 
     # Remove unwanted punctuation and special characters (except useful ones like < > @ # ! ?)
     tweet = re.sub(r'[^\w\s<>@#!?]', '', tweet)
+    # print(tweet + '\n')
 
     # Remove non-printable/control characters (e.g., \n, \t, emojis, corrupted unicode)
     tweet = re.sub(r'[^\x20-\x7E]', '', tweet)
+    # print(tweet + '\n')
 
     # Tokenize the tweet into individual words
     tokens = word_tokenize(tweet)
+    # print(tokens)
+    # print('\n')
 
     # Lemmatize each token to its base form (e.g., "running" → "run")
     lemmatizer = WordNetLemmatizer()
     tokens = [lemmatizer.lemmatize(token) for token in tokens]
+    # print(tokens)
+    # print('\n')
 
     # Remove stopwords, but keep negation-related words (important for sentiment)
     stop_words = set(stopwords.words('english'))
     important_words = {'no', 'not', 'nor', 'neither', 'never', 'nobody', 'none', 'nothing', 'nowhere'}
     stop_words = stop_words - important_words
     tokens = [token for token in tokens if token not in stop_words]
+    # print(tokens)
+    # print('\n')
+
 
     # Reconstruct the cleaned tweet
     return ' '.join(tokens)

deployment/api/tests/test_preprocessing.py

@@ -8,10 +8,10 @@ from preprocessing import preprocess_tweet
 
 def test_preprocess_tweet_removes_hashtags():
     tweet = """
-    Just flew w/ @AirParadis 🚀 & I’m in LOVE! Epic service, comfy seats. 
+    Just flew w/ @AirParadis 🚀 & I’m in LOVE! Epic service, comfy seats, good prices at https://airparadis.com
     #bestflight #AirParadis
     """
-    assert preprocess_tweet(tweet) == "flew w < MENTION > im love ! epic service comfy seat # bestflight # airparadis"
+    assert preprocess_tweet(tweet) == 'flew w < MENTION > im love ! epic service comfy seat good price < URL > # bestflight # airparadis'
 
 def test_preprocess_tweet_removes_mentions():
     tweet = """
@@ -25,4 +25,4 @@ def test_preprocess_tweet_lowercases_text():
     My 3rd time w/ @AirParadis &amp; still amazed &#128525; Crew = lovely &agrave; every step! 
     #flyhappy #AirParadis
     """
-    assert preprocess_tweet(tweet) == '3rd time w < MENTION > still amazed crew lovely every step ! # flyhappy # airparadis'
+    assert preprocess_tweet(tweet) == '3rd time w < MENTION > still amazed crew lovely every step ! # flyhappy # airparadis'