openclassroom-ai/oc-p07-sentiment-api
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

unit tests fixed

Browse source
This commit is contained in:
Grégoire Mureau 2025-07-16 11:39:34 +02:00
parent 98f9dd1729
commit 3a7e603952
2 changed files with 15 additions and 7 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

8
deployment/api/preprocessing.py
View file

@ -28,6 +28,9 @@ def preprocess_tweet(tweet):
if not isinstance(tweet, str):
return ""
# Decode HTML entities (e.g., & → &)
tweet = html.unescape(tweet)
# Convert text to lowercase
tweet = tweet.lower()
@ -40,14 +43,11 @@ def preprocess_tweet(tweet):
# Separate hashtags from the word to preserve semantics during tokenization
tweet = re.sub(r'#(\w+)', r'# \1', tweet)
# Decode HTML entities (e.g., & → &)
tweet = html.unescape(tweet)
# Remove unwanted punctuation and special characters (except useful ones like < > @ # ! ?)
tweet = re.sub(r'[^\w\s<>@#!?]', '', tweet)
# Remove non-printable/control characters (e.g., \n, \t, emojis, corrupted unicode)
# tweet = re.sub(r'[^\x20-\x7E]', '', tweet)
tweet = re.sub(r'[^\x20-\x7E]', '', tweet)
# Tokenize the tweet into individual words
tokens = word_tokenize(tweet)

12
deployment/api/tests/test_preprocessing.py
View file

@ -8,10 +8,10 @@ from preprocessing import preprocess_tweet
def test_preprocess_tweet_removes_hashtags():
tweet = """
Just flew w/ @AirParadis &#128640; &amp; I&#8217;m in LOVE! &Eacute;pic service, comfy seats.
Just flew w/ @AirParadis &#128640; &amp; I&#8217;m in LOVE! Epic service, comfy seats.
#bestflight #AirParadis
"""
assert preprocess_tweet(tweet) == "flew w < MENTION > im love ! épic service comfy seat # bestflight # airparadis"
assert preprocess_tweet(tweet) == "flew w < MENTION > im love ! epic service comfy seat # bestflight # airparadis"
def test_preprocess_tweet_removes_mentions():
tweet = """
@ -26,3 +26,11 @@ def test_preprocess_tweet_lowercases_text():
#flyhappy #AirParadis
"""
assert preprocess_tweet(tweet) == '3rd time w < MENTION > still amazed crew lovely à every step ! # flyhappy # airparadis'
tweet = """
Just flew w/ @AirParadis &#128640; &amp; I&#8217;m in LOVE! Epic service, comfy seats.
#bestflight #AirParadis
"""
preprocess_tweet(tweet)