FastAPI + Streamlit are configured. Only remains the email sending features after 3 wrong predictions reports

deployment/api/app.py

@@ -0,0 +1,85 @@
+import streamlit as st
+import requests
+
+st.title("Analyse de Sentiment - Air Paradis")
+
+tweet = st.text_area("Entrez votre tweet ici :", height=150)
+
+# Initialisation des états
+if "prediction_result" not in st.session_state:
+    st.session_state.prediction_result = None
+
+if "show_report_button" not in st.session_state:
+    st.session_state.show_report_button = False
+
+if "report_sent" not in st.session_state:
+    st.session_state.report_sent = False
+
+if "last_report_sent" not in st.session_state:
+    st.session_state.last_report_sent = False
+
+def reset_state():
+    st.session_state.prediction_result = None
+    st.session_state.show_report_button = False
+    st.session_state.report_sent = False
+    st.session_state.last_report_sent = False
+
+# Bouton d'analyse
+if st.button("Analyser le sentiment"):
+    if not tweet.strip():
+        st.warning("Veuillez entrer un tweet non vide.")
+        reset_state()
+    else:
+        try:
+            response = requests.post(
+                "http://localhost:8000/predict",
+                json={"tweet": tweet}
+            )
+            if response.status_code == 200:
+                data = response.json()
+                st.session_state.prediction_result = data
+                st.session_state.show_report_button = True
+                st.session_state.report_sent = False
+                st.session_state.last_report_sent = False
+            else:
+                st.error(f"Erreur API : {response.status_code} - {response.text}")
+                reset_state()
+        except Exception as e:
+            st.error(f"Erreur lors de la requête vers l'API : {e}")
+            reset_state()
+
+# Affichage du résultat
+if st.session_state.prediction_result:
+    sentiment = st.session_state.prediction_result["sentiment"].capitalize()
+    prob = st.session_state.prediction_result["probability"]
+    st.success(f"Sentiment : {sentiment}")
+    st.info(f"Probabilité : {prob:.2f}")
+
+# Gestion du bouton signalement
+if st.session_state.show_report_button and not st.session_state.report_sent:
+    if st.button("Signaler une mauvaise prédiction"):
+        try:
+            response = requests.post(
+                "http://localhost:8000/report_error",
+                json={"tweet": tweet, "prediction": sentiment}
+            )
+            if response.status_code == 200:
+                data = response.json()
+                # Mettre à jour les états
+                st.session_state.show_report_button = False
+                st.session_state.report_sent = True
+                st.session_state.last_report_sent = data.get("report_sent", False)
+                
+                # Forcer un re-run
+                st.rerun()
+            else:
+                st.error(f"Erreur API : {response.status_code} - {response.text}")
+        except Exception as e:
+            st.error(f"Erreur lors de la requête vers l'API : {e}")
+
+# Message si signalement déjà fait
+elif st.session_state.report_sent:
+    st.success("✅ Merci pour votre signalement.")
+    
+    if st.session_state.last_report_sent:
+        st.info("📩 Un rapport a été envoyé à l'administrateur du site.")

deployment/api/main.py

@@ -1,113 +1,141 @@
 import os
-import dill
+import sys
 import pickle
 from dotenv import load_dotenv
 from fastapi import FastAPI, HTTPException
 from pydantic import BaseModel
 import mlflow
 from mlflow.tracking import MlflowClient
-import mlflow.keras
+from tensorflow.keras.models import load_model
 from tensorflow.keras.preprocessing.sequence import pad_sequences
 
-# Charger les variables d'environnement
+# 🔁 Import de la fonction de prétraitement depuis ../../functions.py
-load_dotenv(dotenv_path='../../.env')
+PARENT_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), "../../"))
+sys.path.append(PARENT_DIR)
+from functions import preprocess_tweet
 
-#%%
+# 🔐 Charger les variables d'environnement (.env à deux niveaux au-dessus)
+load_dotenv(dotenv_path=os.path.join(PARENT_DIR, ".env"))
+
+# 🔗 Configurer MLflow
 mlflow_tracking_uri = os.getenv("MLFLOW_TRACKING_URI")
 if mlflow_tracking_uri:
     mlflow.set_tracking_uri(mlflow_tracking_uri)
 else:
-    print("⚠️ Attention : MLFLOW_TRACKING_URI non défini dans .env")
+    print("⚠️ MLFLOW_TRACKING_URI non défini dans .env")
 
-# Modèle et étape par défaut
+# 📌 Nom du modèle et stage
 MODEL_NAME = "SentimentAnalysisLSTM"
 STAGE = "Production"
 
+# 🧾 Structures des requêtes POST
 class TweetRequest(BaseModel):
     tweet: str
 
-def get_latest_version(client, model_name, stage):
+class ReportRequest(BaseModel):
-    versions = client.search_model_versions(f"name='{model_name}'")
+    tweet: str
-    filtered = [v for v in versions if v.current_stage == stage]
+    prediction: str
-    if not filtered:
-        return None
-    latest = max(filtered, key=lambda v: int(v.version))
-    return latest
 
-def load_model_and_artifacts(model_name="SentimentAnalysisLSTM", stage="Production"):
+# 🧠 Stockage des signalements d’erreurs
+error_reports = {}
+
+# 🔍 Fonction pour charger modèle + tokenizer
+def load_model_and_artifacts(model_name=MODEL_NAME, stage=STAGE):
     client = MlflowClient()
     versions = client.get_latest_versions(model_name, [stage])
     if not versions:
         raise RuntimeError(f"Aucune version du modèle '{model_name}' en stage '{stage}'")
-    version = versions[0]
-    run_id = version.run_id
-    if run_id is None:
-        raise RuntimeError("'run_id' introuvable.")
 
-    # Récupérer les tags du modèle pour trouver embedding_type
+    version_info = versions[0]
-    model_version_info = client.get_model_version(model_name, version.version)
+    run_id = version_info.run_id
+    if not run_id:
+        raise RuntimeError("Run ID introuvable.")
+
+    # Extraire le tag d'embedding
+    model_version_info = client.get_model_version(model_name, version_info.version)
     embedding_type = model_version_info.tags.get("embedding_type", "glove").lower()
 
+    # Déterminer les chemins des artefacts
     local_dir = "./downloaded_artifacts"
     os.makedirs(local_dir, exist_ok=True)
 
-    base_path = "local_artifacts/logs"
+    base_path = "local_artifacts"
+    tokenizer_file = "tokenizer.pickle"
 
-    if embedding_type == "glove":
+    if "glove" in embedding_type:
         keras_file = "final_model_LSTM_GloVe-300d-Fige.keras"
-        tokenizer_file = "tokenizer.pickle"
+    elif "word2vec" in embedding_type:
-        preprocess_file = "preprocess_function.dill"
+        keras_file = "final_model_LSTM_Word2Vec-Fige.keras"
-    elif embedding_type == "word2vec":
-        keras_file = "final_model_LSTM_Word2Vec.keras"
-        tokenizer_file = "tokenizer_word2vec.pickle"
-        preprocess_file = "preprocess_function_word2vec.dill"
     else:
-        raise ValueError(f"Type d'embedding inconnu: {embedding_type}")
+        raise ValueError(f"Embedding type inconnu : {embedding_type}")
 
     def dl(path):
         return client.download_artifacts(run_id=run_id, path=f"{base_path}/{path}", dst_path=local_dir)
 
+    # 🔽 Télécharger les artefacts
     model_path = dl(keras_file)
     tokenizer_path = dl(tokenizer_file)
-    preprocess_path = dl(preprocess_file)
 
-    model = mlflow.keras.load_model(model_path)
+    # 📦 Chargement
+    model = load_model(model_path)
     with open(tokenizer_path, "rb") as f:
         tokenizer = pickle.load(f)
-    with open(preprocess_path, "rb") as f:
-        preprocess_fn = dill.load(f)
-
-    return model, tokenizer, preprocess_fn, embedding_type
 
+    return model, tokenizer, embedding_type
 
+# 🚀 Lancer l'API
 app = FastAPI(title="API Prédiction Sentiment - Air Paradis")
 
 try:
-    model, tokenizer, preprocess_tweet = load_model_and_artifacts()
+    model, tokenizer, embedding_type = load_model_and_artifacts()
 except Exception as e:
-    print(f"❌ Erreur lors du chargement des artefacts : {e}")
+    print(f"\n❌ Erreur lors du chargement des artefacts : {e}")
-    model, tokenizer, preprocess_tweet = None, None, None
+    model, tokenizer = None, None
 
 @app.get("/")
 def root():
-    return {"message": "API Prédiction Sentiment Air Paradis - OK"}
+    return {"message": "API Sentiment - Air Paradis - en ligne"}
 
 @app.post("/predict")
 def predict_sentiment(request: TweetRequest):
-    if model is None or tokenizer is None or preprocess_tweet is None:
+    if model is None or tokenizer is None:
-        raise HTTPException(status_code=503, detail="Modèle ou artefacts non disponibles.")
+        raise HTTPException(status_code=503, detail="Modèle ou artefacts indisponibles.")
+
+    try:
+        processed_tweet = preprocess_tweet(request.tweet)
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=f"Erreur de prétraitement : {e}")
 
-    processed_tweet = preprocess_tweet(request.tweet)
     if not processed_tweet.strip():
         raise HTTPException(status_code=400, detail="Tweet vide ou invalide après prétraitement.")
 
-    sequence = tokenizer.texts_to_sequences([processed_tweet])
-    sequence_padded = pad_sequences(sequence, maxlen=100)  # Ajuster maxlen si besoin
-
     try:
+        sequence = tokenizer.texts_to_sequences([processed_tweet])
+        sequence_padded = pad_sequences(sequence, maxlen=model.input_shape[1])
         prediction_prob = model.predict(sequence_padded)[0][0]
         sentiment = "positif" if prediction_prob >= 0.5 else "negatif"
     except Exception as e:
-        raise HTTPException(status_code=500, detail=f"Erreur lors de l'inférence : {e}")
+        raise HTTPException(status_code=500, detail=f"Erreur d'inférence : {e}")
 
-    return {"sentiment": sentiment, "probability": float(prediction_prob)}
+    return {
+        "sentiment": sentiment,
+        "probability": float(prediction_prob)
+    }
+
+@app.post("/report_error")
+def report_error(request: ReportRequest):
+    tweet = request.tweet.strip()
+    prediction = request.prediction.strip().lower()
+
+    if not tweet or not prediction:
+        raise HTTPException(status_code=400, detail="Tweet ou prédiction manquants.")
+
+    # Ajouter ou mettre à jour le signalement du tweet
+    error_reports[tweet] = prediction
+
+    report_sent = False
+    # Condition : à chaque multiple de 3 signalements totaux
+    if len(error_reports) % 3 == 0:
+        report_sent = True
+
+    return {"report_sent": report_sent}

functions.py

@@ -0,0 +1,72 @@
+import re
+import html
+import numpy as np
+import pandas as pd
+from multiprocessing import Pool
+
+# NLTK
+import nltk
+from nltk.corpus import stopwords
+from nltk.tokenize import word_tokenize
+from nltk.stem import WordNetLemmatizer
+
+# Assure que les ressources nécessaires sont disponibles
+try:
+    stopwords.words('english')
+    word_tokenize("test")
+    WordNetLemmatizer().lemmatize("testing")
+except LookupError:
+    nltk.download('punkt')
+    nltk.download('punkt_tab')
+    nltk.download('stopwords')
+    nltk.download('wordnet')
+    nltk.download('omw-1.4')  # Pour les lemmes WordNet en anglais
+
+# Convertir les labels : 0 pour négatif, 1 pour tout le reste
+def convert_sentiment_label(df):
+    converted_target_data = df.copy()
+    converted_target_data['target'] = converted_target_data['target'].apply(lambda x: 0 if x == 0 else 1)
+    return converted_target_data
+
+# Réduction de la taille du dataset pour équilibrer
+def downsample_data(df, n_samples=50000, random_state=42):
+    negative_samples = df[df['target'] == 0].sample(n=n_samples, random_state=random_state)
+    positive_samples = df[df['target'] == 1].sample(n=n_samples, random_state=random_state)
+    downsampled_data = pd.concat([negative_samples, positive_samples])
+    return downsampled_data
+
+# Fonction principale de prétraitement
+def preprocess_tweet(tweet):
+    if not isinstance(tweet, str):
+        return ""
+
+    tweet = tweet.lower()
+    tweet = re.sub(r'https?://\S+|www\.\S+', '<URL>', tweet)
+    tweet = re.sub(r'@\w+', '<MENTION>', tweet)
+    tweet = re.sub(r'#(\w+)', r'# \1', tweet)
+    tweet = html.unescape(tweet)
+    tweet = re.sub(r'[^\w\s<>@#!?]', '', tweet)
+
+    tokens = word_tokenize(tweet)
+
+    lemmatizer = WordNetLemmatizer()
+    tokens = [lemmatizer.lemmatize(token) for token in tokens]
+
+    stop_words = set(stopwords.words('english'))
+    important_words = {'no', 'not', 'nor', 'neither', 'never', 'nobody', 'none', 'nothing', 'nowhere'}
+    stop_words = stop_words - important_words
+    tokens = [token for token in tokens if token not in stop_words]
+
+    return ' '.join(tokens)
+
+# Traitement en parallèle sur un DataFrame
+def process_in_parallel(df, func, n_jobs=4):
+    df_split = np.array_split(df, n_jobs)
+    with Pool(n_jobs) as pool:
+        df = pd.concat(pool.map(func, df_split))
+    return df
+
+# Application du prétraitement sur une partition
+def apply_preprocessing(df_part):
+    df_part['processed_text'] = df_part['text'].apply(preprocess_tweet)
+    return df_part