main.py udpate
This commit is contained in:
parent
5d983fa9d2
commit
3dea82648c
1 changed files with 72 additions and 64 deletions
|
|
@ -1,77 +1,92 @@
|
|||
import os
|
||||
import dill
|
||||
import pickle
|
||||
from dotenv import load_dotenv
|
||||
import re
|
||||
import html
|
||||
from fastapi import FastAPI, HTTPException
|
||||
from pydantic import BaseModel
|
||||
import mlflow
|
||||
from mlflow.tracking import MlflowClient
|
||||
import mlflow.keras
|
||||
import nltk
|
||||
from nltk.corpus import stopwords
|
||||
from nltk.tokenize import word_tokenize
|
||||
from nltk.stem import WordNetLemmatizer
|
||||
from tensorflow.keras.preprocessing.sequence import pad_sequences
|
||||
|
||||
# Téléchargements nécessaires NLTK
|
||||
nltk.download('punkt')
|
||||
nltk.download('stopwords')
|
||||
nltk.download('wordnet')
|
||||
|
||||
|
||||
# Charger les variables d'environnement depuis le fichier .env à la racine ou accessible
|
||||
# Charger les variables d'environnement
|
||||
load_dotenv(dotenv_path='../../.env')
|
||||
|
||||
# Configuration de MLflow avec les variables d'environnement
|
||||
#%%
|
||||
mlflow_tracking_uri = os.getenv("MLFLOW_TRACKING_URI")
|
||||
mlflow_s3_endpoint_url = os.getenv("MLFLOW_S3_ENDPOINT_URL")
|
||||
aws_access_key_id = os.getenv("AWS_ACCESS_KEY_ID")
|
||||
aws_secret_access_key = os.getenv("AWS_SECRET_ACCESS_KEY")
|
||||
if mlflow_tracking_uri:
|
||||
mlflow.set_tracking_uri(mlflow_tracking_uri)
|
||||
else:
|
||||
print("⚠️ Attention : MLFLOW_TRACKING_URI non défini dans .env")
|
||||
|
||||
# Modèle et étape par défaut
|
||||
MODEL_NAME = "SentimentAnalysisLSTM"
|
||||
STAGE = "Production"
|
||||
|
||||
# Définition du modèle de données pour la requête
|
||||
class TweetRequest(BaseModel):
|
||||
tweet: str
|
||||
|
||||
# Fonction de prétraitement pour les tweets
|
||||
def preprocess_tweet(tweet: str) -> str:
|
||||
"""
|
||||
Prétraite un tweet en appliquant plusieurs transformations :
|
||||
- Conversion en minuscules
|
||||
- Remplacement des URLs, mentions et hashtags par des tokens spéciaux
|
||||
- Suppression des caractères spéciaux
|
||||
- Tokenisation et lemmatisation
|
||||
- Suppression des stopwords
|
||||
"""
|
||||
if not isinstance(tweet, str):
|
||||
return ""
|
||||
def get_latest_version(client, model_name, stage):
|
||||
versions = client.search_model_versions(f"name='{model_name}'")
|
||||
filtered = [v for v in versions if v.current_stage == stage]
|
||||
if not filtered:
|
||||
return None
|
||||
latest = max(filtered, key=lambda v: int(v.version))
|
||||
return latest
|
||||
|
||||
tweet = tweet.lower()
|
||||
tweet = re.sub(r'https?://\S+|www\.\S+', '<URL>', tweet)
|
||||
tweet = re.sub(r'@\w+', '<MENTION>', tweet)
|
||||
tweet = re.sub(r'#(\w+)', r'# \1', tweet)
|
||||
tweet = html.unescape(tweet)
|
||||
tweet = re.sub(r'[^\w\s<>@#!?]', '', tweet)
|
||||
def load_model_and_artifacts(model_name="SentimentAnalysisLSTM", stage="Production"):
|
||||
client = MlflowClient()
|
||||
versions = client.get_latest_versions(model_name, [stage])
|
||||
if not versions:
|
||||
raise RuntimeError(f"Aucune version du modèle '{model_name}' en stage '{stage}'")
|
||||
version = versions[0]
|
||||
run_id = version.run_id
|
||||
if run_id is None:
|
||||
raise RuntimeError("'run_id' introuvable.")
|
||||
|
||||
tokens = word_tokenize(tweet)
|
||||
# Récupérer les tags du modèle pour trouver embedding_type
|
||||
model_version_info = client.get_model_version(model_name, version.version)
|
||||
embedding_type = model_version_info.tags.get("embedding_type", "glove").lower()
|
||||
|
||||
lemmatizer = WordNetLemmatizer()
|
||||
tokens = [lemmatizer.lemmatize(token) for token in tokens]
|
||||
local_dir = "./downloaded_artifacts"
|
||||
os.makedirs(local_dir, exist_ok=True)
|
||||
|
||||
stop_words = set(stopwords.words('english'))
|
||||
important_words = {'no', 'not', 'nor', 'neither', 'never', 'nobody', 'none', 'nothing', 'nowhere'}
|
||||
stop_words = stop_words - important_words
|
||||
tokens = [token for token in tokens if token not in stop_words]
|
||||
base_path = "local_artifacts/logs"
|
||||
|
||||
if embedding_type == "glove":
|
||||
keras_file = "final_model_LSTM_GloVe-300d-Fige.keras"
|
||||
tokenizer_file = "tokenizer.pickle"
|
||||
preprocess_file = "preprocess_function.dill"
|
||||
elif embedding_type == "word2vec":
|
||||
keras_file = "final_model_LSTM_Word2Vec.keras"
|
||||
tokenizer_file = "tokenizer_word2vec.pickle"
|
||||
preprocess_file = "preprocess_function_word2vec.dill"
|
||||
else:
|
||||
raise ValueError(f"Type d'embedding inconnu: {embedding_type}")
|
||||
|
||||
def dl(path):
|
||||
return client.download_artifacts(run_id=run_id, path=f"{base_path}/{path}", dst_path=local_dir)
|
||||
|
||||
model_path = dl(keras_file)
|
||||
tokenizer_path = dl(tokenizer_file)
|
||||
preprocess_path = dl(preprocess_file)
|
||||
|
||||
model = mlflow.keras.load_model(model_path)
|
||||
with open(tokenizer_path, "rb") as f:
|
||||
tokenizer = pickle.load(f)
|
||||
with open(preprocess_path, "rb") as f:
|
||||
preprocess_fn = dill.load(f)
|
||||
|
||||
return model, tokenizer, preprocess_fn, embedding_type
|
||||
|
||||
return ' '.join(tokens)
|
||||
|
||||
# Initialisation FastAPI
|
||||
app = FastAPI(title="API Prédiction Sentiment - Air Paradis")
|
||||
|
||||
# Chargement du modèle MLflow (URI fixe du modèle en stage Production)
|
||||
MODEL_URI = "models:/SentimentAnalysisLSTM/Production"
|
||||
try:
|
||||
model = mlflow.keras.load_model(MODEL_URI)
|
||||
print("Modèle chargé depuis MLflow avec succès.")
|
||||
model, tokenizer, preprocess_tweet = load_model_and_artifacts()
|
||||
except Exception as e:
|
||||
print(f"Erreur lors du chargement du modèle MLflow : {e}")
|
||||
model = None
|
||||
print(f"❌ Erreur lors du chargement des artefacts : {e}")
|
||||
model, tokenizer, preprocess_tweet = None, None, None
|
||||
|
||||
@app.get("/")
|
||||
def root():
|
||||
|
|
@ -79,27 +94,20 @@ def root():
|
|||
|
||||
@app.post("/predict")
|
||||
def predict_sentiment(request: TweetRequest):
|
||||
if model is None:
|
||||
raise HTTPException(status_code=503, detail="Modèle non disponible.")
|
||||
if model is None or tokenizer is None or preprocess_tweet is None:
|
||||
raise HTTPException(status_code=503, detail="Modèle ou artefacts non disponibles.")
|
||||
|
||||
processed_tweet = preprocess_tweet(request.tweet)
|
||||
if processed_tweet == "":
|
||||
if not processed_tweet.strip():
|
||||
raise HTTPException(status_code=400, detail="Tweet vide ou invalide après prétraitement.")
|
||||
|
||||
# Ici, il faudra adapter selon la manière dont ton modèle attend l'entrée
|
||||
# Par exemple, si ton modèle attend une séquence d'indices, il faut intégrer la vectorisation / tokenisation
|
||||
# Pour l'exemple, on fait un dummy wrap en liste
|
||||
input_data = [processed_tweet]
|
||||
sequence = tokenizer.texts_to_sequences([processed_tweet])
|
||||
sequence_padded = pad_sequences(sequence, maxlen=100) # Ajuster maxlen si besoin
|
||||
|
||||
# Modèle Keras attend généralement un array numpy, ou séquence tokenisée
|
||||
# À adapter selon ta pipeline exacte
|
||||
import numpy as np
|
||||
try:
|
||||
# Exemple naïf : on prédit directement sur la liste textuelle (à adapter !)
|
||||
prediction_prob = model.predict(input_data)[0][0] # supposons sortie [prob_neg]
|
||||
prediction_prob = model.predict(sequence_padded)[0][0]
|
||||
sentiment = "positif" if prediction_prob >= 0.5 else "negatif"
|
||||
except Exception as e:
|
||||
raise HTTPException(status_code=500, detail=f"Erreur lors de l'inférence : {e}")
|
||||
|
||||
return {"sentiment": sentiment, "probability": float(prediction_prob)}
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue