66 lines
No EOL
2.2 KiB
Python
66 lines
No EOL
2.2 KiB
Python
import re
|
|
import html
|
|
import nltk
|
|
from nltk.corpus import stopwords
|
|
from nltk.tokenize import word_tokenize
|
|
from nltk.stem import WordNetLemmatizer
|
|
|
|
# Ensure that the necessary resources are available
|
|
try:
|
|
stopwords.words('english')
|
|
word_tokenize("test")
|
|
WordNetLemmatizer().lemmatize("testing")
|
|
except LookupError:
|
|
nltk.download('punkt')
|
|
nltk.download('punkt_tab')
|
|
nltk.download('stopwords')
|
|
nltk.download('wordnet')
|
|
nltk.download('omw-1.4')
|
|
|
|
# PREPROCESSING FUNCTION FOR CLASSIC AND ADVANCED MODELS
|
|
def preprocess_tweet(tweet):
|
|
"""
|
|
Preprocesses a tweet for classic ML models or word embedding models (e.g., Word2Vec, GloVe).
|
|
The text is cleaned, normalized, lemmatized, and stripped of unnecessary noise.
|
|
"""
|
|
|
|
# Ensure input is a string
|
|
if not isinstance(tweet, str):
|
|
return ""
|
|
|
|
# Decode HTML entities (e.g., & → &)
|
|
tweet = html.unescape(tweet)
|
|
|
|
# Convert text to lowercase
|
|
tweet = tweet.lower()
|
|
|
|
# Replace URLs with a generic token
|
|
tweet = re.sub(r'https?://\S+|www\.\S+', '<URL>', tweet)
|
|
|
|
# Replace mentions (@username) with a generic token
|
|
tweet = re.sub(r'@\w+', '<MENTION>', tweet)
|
|
|
|
# Separate hashtags from the word to preserve semantics during tokenization
|
|
tweet = re.sub(r'#(\w+)', r'# \1', tweet)
|
|
|
|
# Remove unwanted punctuation and special characters (except useful ones like < > @ # ! ?)
|
|
tweet = re.sub(r'[^\w\s<>@#!?]', '', tweet)
|
|
|
|
# Remove non-printable/control characters (e.g., \n, \t, emojis, corrupted unicode)
|
|
tweet = re.sub(r'[^\x20-\x7E]', '', tweet)
|
|
|
|
# Tokenize the tweet into individual words
|
|
tokens = word_tokenize(tweet)
|
|
|
|
# Lemmatize each token to its base form (e.g., "running" → "run")
|
|
lemmatizer = WordNetLemmatizer()
|
|
tokens = [lemmatizer.lemmatize(token) for token in tokens]
|
|
|
|
# Remove stopwords, but keep negation-related words (important for sentiment)
|
|
stop_words = set(stopwords.words('english'))
|
|
important_words = {'no', 'not', 'nor', 'neither', 'never', 'nobody', 'none', 'nothing', 'nowhere'}
|
|
stop_words = stop_words - important_words
|
|
tokens = [token for token in tokens if token not in stop_words]
|
|
|
|
# Reconstruct the cleaned tweet
|
|
return ' '.join(tokens) |