preprocess function order changed
This commit is contained in:
parent
4e23a08f74
commit
a47fe40b78
1 changed files with 3 additions and 3 deletions
|
|
@ -65,6 +65,9 @@ def preprocess_tweet(tweet):
|
|||
if not isinstance(tweet, str):
|
||||
return ""
|
||||
|
||||
# Decode HTML entities (e.g., & to &)
|
||||
tweet = html.unescape(tweet)
|
||||
|
||||
# Convert to lowercase
|
||||
tweet = tweet.lower()
|
||||
|
||||
|
|
@ -77,9 +80,6 @@ def preprocess_tweet(tweet):
|
|||
# Separate hashtags from words to facilitate tokenization
|
||||
tweet = re.sub(r'#(\w+)', r'# \1', tweet)
|
||||
|
||||
# Decode HTML entities (e.g., & to &)
|
||||
tweet = html.unescape(tweet)
|
||||
|
||||
# Remove unwanted punctuation and special characters except some useful ones
|
||||
tweet = re.sub(r'[^\w\s<>@#!?]', '', tweet)
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue