From d3b0af612677aa18bfb1312dd07bd522ce6738fb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gr=C3=A9goire=20MUREAU?= Date: Wed, 16 Jul 2025 11:12:22 +0200 Subject: [PATCH] functions in english --- deployment/api/preprocessing.py | 38 +++++++++--- functions.py | 80 +++++++++++++++++++++---- presentation/récapitulatif modèle.xlsx | Bin 0 -> 6542 bytes 3 files changed, 97 insertions(+), 21 deletions(-) create mode 100644 presentation/récapitulatif modèle.xlsx diff --git a/deployment/api/preprocessing.py b/deployment/api/preprocessing.py index 7f822e8..1bed0c2 100644 --- a/deployment/api/preprocessing.py +++ b/deployment/api/preprocessing.py @@ -5,7 +5,7 @@ from nltk.corpus import stopwords from nltk.tokenize import word_tokenize from nltk.stem import WordNetLemmatizer -# Assurer que les ressources nécessaires sont disponibles +# Ensure that the necessary resources are available try: stopwords.words('english') word_tokenize("test") @@ -17,28 +17,50 @@ except LookupError: nltk.download('wordnet') nltk.download('omw-1.4') +# PREPROCESSING FUNCTION FOR CLASSIC AND ADVANCED MODELS def preprocess_tweet(tweet): """ - Tweet pre-processing function for the Sentiment Analysis API + Preprocesses a tweet for classic ML models or word embedding models (e.g., Word2Vec, GloVe). + The text is cleaned, normalized, lemmatized, and stripped of unnecessary noise. """ + + # Ensure input is a string if not isinstance(tweet, str): return "" - tweet = html.unescape(tweet) + + # Convert text to lowercase tweet = tweet.lower() + + # Replace URLs with a generic token tweet = re.sub(r'https?://\S+|www\.\S+', '', tweet) + + # Replace mentions (@username) with a generic token tweet = re.sub(r'@\w+', '', tweet) + + # Separate hashtags from the word to preserve semantics during tokenization tweet = re.sub(r'#(\w+)', r'# \1', tweet) + + # Decode HTML entities (e.g., & → &) + tweet = html.unescape(tweet) + + # Remove unwanted punctuation and special characters (except useful ones like < > @ # ! ?) tweet = re.sub(r'[^\w\s<>@#!?]', '', tweet) - + + # Remove non-printable/control characters (e.g., \n, \t, emojis, corrupted unicode) + tweet = re.sub(r'[^\x20-\x7E]', '', tweet) + + # Tokenize the tweet into individual words tokens = word_tokenize(tweet) - + + # Lemmatize each token to its base form (e.g., "running" → "run") lemmatizer = WordNetLemmatizer() tokens = [lemmatizer.lemmatize(token) for token in tokens] - + + # Remove stopwords, but keep negation-related words (important for sentiment) stop_words = set(stopwords.words('english')) important_words = {'no', 'not', 'nor', 'neither', 'never', 'nobody', 'none', 'nothing', 'nowhere'} stop_words = stop_words - important_words - tokens = [token for token in tokens if token not in stop_words] - + + # Reconstruct the cleaned tweet return ' '.join(tokens) \ No newline at end of file diff --git a/functions.py b/functions.py index 7663a65..4346fe4 100644 --- a/functions.py +++ b/functions.py @@ -4,68 +4,122 @@ import numpy as np import pandas as pd from multiprocessing import Pool -# NLTK +# NLTK imports for NLP preprocessing import nltk from nltk.corpus import stopwords from nltk.tokenize import word_tokenize from nltk.stem import WordNetLemmatizer -# Assure que les ressources nécessaires sont disponibles +# Ensure necessary NLTK resources are downloaded and available try: stopwords.words('english') word_tokenize("test") WordNetLemmatizer().lemmatize("testing") except LookupError: - nltk.download('punkt') - nltk.download('punkt_tab') - nltk.download('stopwords') - nltk.download('wordnet') - nltk.download('omw-1.4') # Pour les lemmes WordNet en anglais + nltk.download('punkt') # Tokenizer models + nltk.download('punkt_tab') # Additional tokenizer support + nltk.download('stopwords') # Stopwords lists + nltk.download('wordnet') # WordNet lexicon for lemmatization + nltk.download('omw-1.4') # WordNet lemmas data -# Convertir les labels : 0 pour négatif, 1 pour tout le reste +# Convert sentiment labels to binary: 0 for negative, 1 for all other classes def convert_sentiment_label(df): + """ + Convert the 'target' column in the DataFrame to binary sentiment labels: + - 0 for negative sentiment (label 0) + - 1 for any other sentiment label + Returns a new DataFrame with updated labels. + """ converted_target_data = df.copy() converted_target_data['target'] = converted_target_data['target'].apply(lambda x: 0 if x == 0 else 1) return converted_target_data -# Réduction de la taille du dataset pour équilibrer +# Downsample the dataset to balance classes by limiting samples per class def downsample_data(df, n_samples=50000, random_state=42): + """ + Perform downsampling to balance dataset classes. + Selects `n_samples` examples from both negative (0) and positive (1) classes. + Returns a balanced DataFrame with equal number of samples per class. + """ negative_samples = df[df['target'] == 0].sample(n=n_samples, random_state=random_state) positive_samples = df[df['target'] == 1].sample(n=n_samples, random_state=random_state) downsampled_data = pd.concat([negative_samples, positive_samples]) return downsampled_data -# Fonction principale de prétraitement +# PREPROCESSING FUNCTION FOR CLASSIC AND ADVANCED MODELS def preprocess_tweet(tweet): + """ + Preprocess a tweet for classic ML or word embedding models (e.g., Word2Vec, GloVe). + Steps: + - Normalize text (lowercase) + - Replace URLs and mentions with tokens + - Decode HTML entities + - Remove unwanted special characters and non-printable chars + - Tokenize text + - Lemmatize tokens + - Remove stopwords except negation words + Returns cleaned, normalized text as a single string. + """ + + # Ensure input is a string if not isinstance(tweet, str): return "" - tweet = html.unescape(tweet) + + # Convert to lowercase tweet = tweet.lower() + + # Replace URLs with placeholder token tweet = re.sub(r'https?://\S+|www\.\S+', '', tweet) + + # Replace user mentions with placeholder token tweet = re.sub(r'@\w+', '', tweet) + + # Separate hashtags from words to facilitate tokenization tweet = re.sub(r'#(\w+)', r'# \1', tweet) + + # Decode HTML entities (e.g., & to &) + tweet = html.unescape(tweet) + + # Remove unwanted punctuation and special characters except some useful ones tweet = re.sub(r'[^\w\s<>@#!?]', '', tweet) + # Remove non-printable and control characters (e.g., tabs, newlines, emojis) + tweet = re.sub(r'[^\x20-\x7E]', '', tweet) + + # Tokenize tweet into words tokens = word_tokenize(tweet) + # Lemmatize tokens to their base form lemmatizer = WordNetLemmatizer() tokens = [lemmatizer.lemmatize(token) for token in tokens] + # Remove stopwords but keep negation-related words to preserve sentiment meaning stop_words = set(stopwords.words('english')) important_words = {'no', 'not', 'nor', 'neither', 'never', 'nobody', 'none', 'nothing', 'nowhere'} stop_words = stop_words - important_words tokens = [token for token in tokens if token not in stop_words] + # Rebuild cleaned tweet text return ' '.join(tokens) -# Traitement en parallèle sur un DataFrame +# Parallel processing of a DataFrame using multiple CPU cores def process_in_parallel(df, func, n_jobs=4): + """ + Split a DataFrame into `n_jobs` parts and apply `func` to each part in parallel, + using multiprocessing Pool for speed-up. + Returns the concatenated processed DataFrame. + """ df_split = np.array_split(df, n_jobs) with Pool(n_jobs) as pool: df = pd.concat(pool.map(func, df_split)) return df -# Application du prétraitement sur une partition +# Apply preprocessing function on a DataFrame partition def apply_preprocessing(df_part): + """ + Apply tweet preprocessing to the 'text' column of a DataFrame partition. + Stores results in a new column 'processed_text'. + Returns the updated DataFrame partition. + """ df_part['processed_text'] = df_part['text'].apply(preprocess_tweet) return df_part diff --git a/presentation/récapitulatif modèle.xlsx b/presentation/récapitulatif modèle.xlsx new file mode 100644 index 0000000000000000000000000000000000000000..00a62c7d418e18e38e6d8fcd1e62bacb243cc77b GIT binary patch literal 6542 zcmeHMWmuHm)*b}uZggl+q)U4Yl40RYGd0018Bj<%?c zwS%#>gTAV(t+Bl>vx}8w$%j6Lc2;chQ>iP_Ri==+N+IOuJHFkx4RqwMm!%EISX^g^ zuley#KM7?Lm~(wHgwERYbENgZ7{|8=DrhdAOpMfkD?5>CtT9Xvtv!e)NX$_gc0nST zc~CiZbYfCDiP+{_N&2cp_8A|Fk0a76s#7>%DyM)}(gs%z6~yhCHuh=ukY7Qw>>?=1 z4dX>>lMuD~6N2)`)AQWHucHW{jcsXaY$U!z)pDz#6S(z96A;n_5g=USfv zu~i@W!Ap8KdtLR;%<+Z)fKZ++bC!hS;`371IF|+_T-Z4x(LVvlOW2hxb>KLS9gm$P zNy@m=IW!x$3{MY&#T7 zZpHiM!K9D^0a0<{p0PtEyJ<6yy|t=1_xn;^JGfBViy?bGzM;WvE$Q>hLxJao(b_9#@@<|vt9F+<&db@if*UL zNDRSZReAV~>b94s`+fw0RPR`?8_Y%FD=oNj`ZnOj*x!kRY&AZ`)Ae&p&WYsPetew} z)zz(_tks2aDsRL&>(ac_hSSQ}$AHQEKwZ=mM-M^sRm$m6YgYAypP+}vuIK^7Q}IX3 zp2*{-;)N>bJsTTCT&fT@? zL?dL4sZ6l4Bh%LSO%FZ^9X=O#>j_~&HQ3Yoq$i#YUc{bTDG9Dkt#o=yiw+E{1|W3dqkmVj`t<dh1ga=OBF2WQr!Gg>q z*y>`*@>iU3wgFoh+Spj!9;UxyPIRC7?@<5#+Aq!c6B|`zf{!~FU`g;aMQ~$JVY){RTui4j-t?Y%LbB+O3 zfbRKO#jl(%Qal~HDM%Vr}#;GCA40oQ2-aluT%0j(*tl5Krw1N{3jY42ZM z0PXZbEigOzNn1SmY$a)myDK|iDiV_LU9KJw-MQag?n@8avWkacf`AABVEl!Py@RWz zvHdL`IjY(gGpzVtI>=%;A#2p+h`Q{KAKUzmNyq;4mD$1fWP(?6zl z0dzx-1F|PETQQ6%Oi(GnB~K0=rNQjewEXHPrA*?>O_Yg|?<|T1~Pj>G0_UyAhE8GVTp^P0LwC*u-WO$RY zG+FEm%$Kc?RgcKv)^F*}QISAjec`r4@{=5%zc4`eEe)*Sb+~7Q`(D_NMBp;Ej|r0J-CRN!5&&St z{8#Y?JJ=4U##Y9E&1|=Bysaf0Ir9R$=nhD%O;>7{P zG`qfB@AFGm)G<#nOvhWR?mwuGF?((EHLoMNwjU{5IMd*f4uwq!-Jxxtj)C=j)BY4( z12b^j5Wa)oi?mu!GN|*|Fj{euZFyqyf%~@?MlGQT9>?pQg^`1R@!U?CLRE%_AeX!) zN+PA*L6bMW;nDt_HXbHz7NwbrOG!^%&N$7Jl)UK(48#RuVm(*Uu=AdN*I@Q|#JA2c zpW9*=V%mq0oVRW1pN>3|5rk%Hnv}a}OsKZ$XNX8skSU@rFVacjVq7@sizWe5whESj z{5p5x!0)Rp)#611yT#yQl_>D{`EE~KR(mm_#e~GZ6w2yB@Q4$Y7zQ0W1(5DC3;=V2c8cciW@)TH3IJ8FT#FB4*3h`)BMB2IN6vZOu(qAQb zzuGhFOCM5gqk~HzGah~W_4G8!rw1|p>Zq$t-aPRt}*2n{=D+!N~Os2xnJ z10S$$ysl5u$y?d4uXpmfpsjtMI565i0Ls08T^y6MK#c*ck5qpBxFjm~_TK*yXmfq> zy*}c)CbCQV9LJ80>*pfJih#Wj87b!8q_<>Hv|rEzAK?W*h8JT%a$|BmyFGcHQ z7ZtC~rOBAd8CFPnu0hzOMmD@ePEm{C-;jL>hQ3TjS;1lB3EIFBq%ty& z@1UynymS?>@h9;*od|TJpaNsZ`^N>}ugwd1fUPS?8j{ zS$(uZN9`kNq&}x;OsU7&aW`1mwlz#D}JXM&*ePBUv z%J>YW`|udAY9fwIprQCa4pcyWQAti7=^I0UzNB{*H`bup{@SYTQd@Wb z5$gC)A^7Zd0aR5I0X%)&C3MoNcj&4Lsa<8qvM6b0wZrWite8mT+!VoON$@>nCt5Ns zek2Z?WS`Akkad{*;1?Jl#E3*wNKz3pPF?ugPEw0_RyPF1hQX5DK_E@z0N(lv2_e)K zl3#Uv@RA?vIeoqwh7;A>j+Al79OkqU3b4RKP_m7vxhSU1LQ@?8)J`!tfCg^(D*bG` zR6EbgNBT8B<6_WqVLz^>V`d70viWoxZ#sD1ZL2VghHsMjr}OqW&_d?t=M9A?sc6W7 zQv12zncj~amOP}-7uGRBFp5bvHJue!0Wt}Iw&5$>>XV9HBd)URL=k`JIn`*Ea&dMr zQ>e1A*-NXPSEu(!&kt5cxpiswm8e-srQubXCUKm>p$ashL9%qndMJ4UEdY2+>-c&Z zMv~9#Fzu%p&Ro5`JxR{w$GP1Z#@|E5vFx6va0uTti6LC*QdQU}E!W7AAWgy1oagAR z{lEsPl?kOfb&gF-H`berG&^^yk*OP$7%M6Usq-Z-1w*VcDdL!CD2tDL*y)XMVhCwQ zEsY9HENi`yNvB?k#aGbE9b|Iwb96auE=WdoZo6qpEXCTM%{2f6R%6@3)8xm9^Xo<< z)baxq=M7MXdj&EfATc7$kQ%S)nOLPMOvJX#$EcN!odQtC(v>xe1Jy9q6+(c2p9?b< ziIc?6bw^pFs+7A^h7UKd7Pa7;AD}=c_cHx=B)crhmrfVMzN?k;nZKBhhpyH< zyhLi8pUufSTM=}YW-a0%$YB}o4_GHA;kMqXJ8r}eL)*zykS5+#L32+X7$%Ffmh)x3 z)`+mcEqjUdTqa7EXbth)tsf^}T0KQn&hfoMzfKdw{Egl-VFmfOVHOG>NF*(a9pF}_ zf=Ki0ShtwBt0angwN2j4to8HtXC5fqa%eVlp%Z@EH!7%bvmLRPei#Wx;tLFI74Qx3 zWtHl(+r)qItXS_1qH&R9EWk8nRjmD~@ja;hCs9d}8a?N>Q}wr+9rJpd1leYV@0#Y1 z9VvjdmVt4~``9VlraGOrRwu^dHOmvS76cEg6x$U+SXKI`ZNdlK_Qk90g$ns*NZlu}g5b6< zzNHmlY5(t)e94Ca>ls$`?p;q*VQ0yjl{)Fy`5N)~Xsqb%QPpq7 z(CEY0xCQJ3cL}+sh9=tw6X73t?|^k&^`R(sCCecV(S`K^$-^4r{J@>Gx@oXhU5I@- zO>jFaZ3t?(``6i)K!aWoYYTgcvd$%$VSNr+rE^@toRG>k|4Z?m%tCk*Rg9q2sITd~ zbP^#0T<%tkbJ9wlAy42*YHYsAuv(-%`-QBOqj(TLPe-FXDyuh3p`rPV#(P6H(p1+w zvj~HhEoSuGk&=%Eobq)Dc#h_0^~0E&W>!KPH!zV8J}}Rtx?qvkDrZ52H-LDLNB%2a z1E0<1DaF~)@lRa^_^j&ek?e2pxz~vEmOZKkoz^6k&*;1DfcdryGXi#>%K5}XM&5u? z#Pq~_8=lo+&X7d?q_>3%Ya41<#?Ud*0u_4#>D?2qR@zHRVXE566pDwZ))F~lh3y04}v4`ynSh^o?9Dl{|qxNpJ5PO)EWLb!cK`w#KHovibWV(2Ak@W`26( z7F5ePVDeplP@I6lc=GYs4=J;Bcd=IwI&?PETs0nju!@8O3&TsCkAP6yaPtONDJeNe zau)~WHNDv3bVff{$nCNfSfzF{O^LQaIAXrR%bwReiwanLX$eG&Itg5w1`+!gq&j3`mAwL3rixI*h;qsImnZ9y}-E$;D!OrhFJPO#k`y@B>7 z3z}8lp_y1KknE6HX8F4i+o<5TY-7te+FIcj1;HmqCw{=Va{qdrq%h4%`M$GwRX-*m zpLhO>x`QrMWKO2rsJw5js?*ZA&X<6z_9mg~VL<&?I35rG(fY)=Zl~Vn5&8iQO$gTr zE2{H@i4M6X;PC;nQoJr9d6jwa84$UdJ;Ay98dkC2S?muH8&#GVC@u z?0}Yw(!+VG>!!OY3T!nFSiX}G-W;OZUgYJgk?o2nM?y~Tp4#zsKL8qZ;LHL<`LUHF z-#7^U$S%|ZWE0b4Q>1oE$1Vz%!&7e$1iT{Qe(3ym$N9%f#8jBm@}hvI0P$8s#1FH_ z?dytr>gN~pWsks9vmp=Y*(FrW-@P;riK#e_Cad$H3+~^OGAq--*^=?zV?sJ{U}pEo z?lMhorhb6t@v*AUOIaWv%98w?{)rts+KMYN^hFPqeEjCoej|zN*h0=0Y8Lqw%Q9H3 z3D7V*A((ac@DJp0510UV48VQ%8;1Me#(&_#m1O^^;68tQC+!Wh<=@!cKZ3V0{FeH< z&u!ibrobHF-^TwJ)A^V5e<#lFq&;Co+kG1C&u9N}wflU~o%AXUr~l3l-Pd!UKe^L$ z1!GiTdhXbiKcatr{{7DNPP7^2x9I&o_P&DqIq*(F73}of%ZT@d??=Zw;R)D(Apa8n qU$OF!mOn7wKe2aOf(ia+!awI%N%kQO6#xJgc6q>Td_{Qs?tcJg+`qU0 literal 0 HcmV?d00001