data_preprocessing.py

"""
data_preprocessing_v2.py
Module PRODUCTION de prétraitement des données ML
Version améliorée avec fit/transform pattern, sans data leakage
Compatible avec base_model.py et data_loader.py
"""

import pandas as pd
import numpy as np
from typing import Dict, List, Tuple, Optional, Union, Any
from datetime import datetime
import warnings
import json
import pickle
from pathlib import Path
from copy import deepcopy

warnings.filterwarnings('ignore')

# Imports sklearn
from sklearn.preprocessing import (
    StandardScaler, MinMaxScaler, RobustScaler, MaxAbsScaler,
    LabelEncoder, OneHotEncoder, OrdinalEncoder,
    PowerTransformer, QuantileTransformer, Normalizer,
    PolynomialFeatures, KBinsDiscretizer
)
from sklearn.impute import SimpleImputer, KNNImputer
from sklearn.feature_selection import (
    SelectKBest, SelectPercentile, RFE, SelectFromModel,
    VarianceThreshold, chi2, f_classif, f_regression, 
    mutual_info_classif, mutual_info_regression
)
from sklearn.decomposition import PCA, TruncatedSVD
from sklearn.model_selection import train_test_split
from sklearn.utils import resample
from sklearn.base import BaseEstimator, TransformerMixin
import scipy.stats as stats


# ============================================================================
# EXCEPTIONS PERSONNALISÉES
# ============================================================================

class PreprocessorNotFittedError(Exception):
    """Erreur quand le preprocessor n'est pas fitted"""
    pass

class DataValidationError(Exception):
    """Erreur de validation des données"""
    pass

class TransformationError(Exception):
    """Erreur durant une transformation"""
    pass


# ============================================================================
# TRANSFORMERS PERSONNALISÉS (Compatible sklearn)
# ============================================================================

class SmartEncoder(BaseEstimator, TransformerMixin):
    """
    Encodeur intelligent qui choisit automatiquement la meilleure stratégie
    Compatible avec sklearn pipeline
    """
    
    def __init__(self, max_categories_onehot: int = 10, handle_unknown: str = 'ignore'):
        self.max_categories_onehot = max_categories_onehot
        self.handle_unknown = handle_unknown
        self.encoders_ = {}
        self.encoding_strategies_ = {}
        self.feature_names_out_ = []
        
    def fit(self, X: pd.DataFrame, y=None):
        """Apprend les encodages"""
        X = X.copy()
        cat_cols = X.select_dtypes(include=['object', 'category']).columns
        
        for col in cat_cols:
            n_unique = X[col].nunique()
            
            if n_unique == 2:
                # Binaire → Label Encoding
                self.encoding_strategies_[col] = 'label'
                le = LabelEncoder()
                le.fit(X[col].astype(str))
                self.encoders_[col] = le
                
            elif n_unique <= self.max_categories_onehot:
                # One-Hot Encoding
                self.encoding_strategies_[col] = 'onehot'
                ohe = OneHotEncoder(sparse_output=False, handle_unknown=self.handle_unknown)
                ohe.fit(X[[col]].astype(str))
                self.encoders_[col] = ohe
                
            else:
                # Frequency Encoding
                self.encoding_strategies_[col] = 'frequency'
                freq_map = X[col].value_counts(normalize=True).to_dict()
                # Ajouter une valeur par défaut pour unseen categories
                freq_map['__UNKNOWN__'] = 0.0
                self.encoders_[col] = freq_map
        
        # Calculer feature_names_out
        self._calculate_feature_names(X)
        
        return self
    
    def transform(self, X: pd.DataFrame) -> pd.DataFrame:
        """Applique les encodages appris"""
        X = X.copy()
        
        for col, strategy in self.encoding_strategies_.items():
            if col not in X.columns:
                continue
                
            if strategy == 'label':
                le = self.encoders_[col]
                # Gérer les valeurs inconnues
                X[col] = X[col].astype(str).apply(
                    lambda x: x if x in le.classes_ else le.classes_[0]
                )
                X[col] = le.transform(X[col])
                
            elif strategy == 'onehot':
                ohe = self.encoders_[col]
                encoded = ohe.transform(X[[col]].astype(str))
                feature_names = ohe.get_feature_names_out([col])
                
                # Créer DataFrame avec noms cohérents
                encoded_df = pd.DataFrame(
                    encoded, 
                    columns=feature_names,
                    index=X.index
                )
                X = pd.concat([X.drop(columns=[col]), encoded_df], axis=1)
                
            elif strategy == 'frequency':
                freq_map = self.encoders_[col]
                X[col] = X[col].map(freq_map).fillna(freq_map['__UNKNOWN__'])
        
        # Assurer que toutes les colonnes attendues sont présentes
        for col in self.feature_names_out_:
            if col not in X.columns:
                X[col] = 0  # Colonne manquante = 0
        
        # Retourner dans le bon ordre
        return X[self.feature_names_out_]
    
    def _calculate_feature_names(self, X: pd.DataFrame):
        """Calcule les noms de features après transformation"""
        X_temp = X.copy()
        
        for col, strategy in self.encoding_strategies_.items():
            if strategy == 'onehot':
                ohe = self.encoders_[col]
                feature_names = list(ohe.get_feature_names_out([col]))
                X_temp = X_temp.drop(columns=[col])
                for fname in feature_names:
                    X_temp[fname] = 0
            # label et frequency gardent le même nom
        
        self.feature_names_out_ = list(X_temp.columns)
    
    def get_feature_names_out(self, input_features=None):
        """Retourne les noms de features de sortie"""
        return self.feature_names_out_


class OutlierHandler(BaseEstimator, TransformerMixin):
    """Gestionnaire d'outliers compatible sklearn"""
    
    def __init__(self, method: str = 'clip', strategy: str = 'iqr', 
                 columns: List[str] = None):
        self.method = method
        self.strategy = strategy
        self.columns = columns
        self.bounds_ = {}
        
    def fit(self, X: pd.DataFrame, y=None):
        """Calcule les bornes sur les données d'entraînement"""
        X = X.copy()
        cols = self.columns or X.select_dtypes(include=[np.number]).columns
        
        for col in cols:
            if col not in X.columns:
                continue
                
            if self.strategy == 'iqr':
                Q1 = X[col].quantile(0.25)
                Q3 = X[col].quantile(0.75)
                IQR = Q3 - Q1
                lower_bound = Q1 - 1.5 * IQR
                upper_bound = Q3 + 1.5 * IQR
                
            elif self.strategy == 'zscore':
                mean = X[col].mean()
                std = X[col].std()
                lower_bound = mean - 3 * std
                upper_bound = mean + 3 * std
            
            self.bounds_[col] = (lower_bound, upper_bound)
        
        return self
    
    def transform(self, X: pd.DataFrame) -> pd.DataFrame:
        """Applique le traitement des outliers"""
        X = X.copy()
        
        if self.method == 'clip':
            for col, (lower, upper) in self.bounds_.items():
                if col in X.columns:
                    X[col] = X[col].clip(lower=lower, upper=upper)
        
        elif self.method == 'remove':
            # Marquer les outliers
            mask = pd.Series([True] * len(X), index=X.index)
            for col, (lower, upper) in self.bounds_.items():
                if col in X.columns:
                    mask &= (X[col] >= lower) & (X[col] <= upper)
            X = X[mask]
        
        return X


class SmartImputer(BaseEstimator, TransformerMixin):
    """Imputer intelligent qui choisit la stratégie selon le type"""
    
    def __init__(self, numeric_strategy: str = 'median', 
                 categorical_strategy: str = 'most_frequent',
                 threshold_drop: float = 0.5):
        self.numeric_strategy = numeric_strategy
        self.categorical_strategy = categorical_strategy
        self.threshold_drop = threshold_drop
        self.imputers_ = {}
        self.columns_to_drop_ = []
        
    def fit(self, X: pd.DataFrame, y=None):
        """Apprend les stratégies d'imputation"""
        X = X.copy()
        
        # Identifier colonnes à supprimer (trop de missing)
        missing_pct = X.isnull().sum() / len(X)
        self.columns_to_drop_ = missing_pct[missing_pct > self.threshold_drop].index.tolist()
        
        # Fit imputers pour les colonnes restantes
        X_remaining = X.drop(columns=self.columns_to_drop_)
        
        # Numeric imputer
        numeric_cols = X_remaining.select_dtypes(include=[np.number]).columns
        if len(numeric_cols) > 0:
            if self.numeric_strategy == 'knn':
                self.imputers_['numeric'] = KNNImputer(n_neighbors=5)
            else:
                self.imputers_['numeric'] = SimpleImputer(strategy=self.numeric_strategy)
            
            self.imputers_['numeric'].fit(X_remaining[numeric_cols])
            self.imputers_['numeric_cols'] = list(numeric_cols)
        
        # Categorical imputer
        cat_cols = X_remaining.select_dtypes(include=['object', 'category']).columns
        if len(cat_cols) > 0:
            self.imputers_['categorical'] = SimpleImputer(strategy=self.categorical_strategy)
            self.imputers_['categorical'].fit(X_remaining[cat_cols].astype(str))
            self.imputers_['categorical_cols'] = list(cat_cols)
        
        return self
    
    def transform(self, X: pd.DataFrame) -> pd.DataFrame:
        """Applique l'imputation"""
        X = X.copy()
        
        # Supprimer colonnes avec trop de missing
        X = X.drop(columns=[col for col in self.columns_to_drop_ if col in X.columns])
        
        # Imputer numeric
        if 'numeric' in self.imputers_:
            numeric_cols = [col for col in self.imputers_['numeric_cols'] if col in X.columns]
            if numeric_cols:
                X[numeric_cols] = self.imputers_['numeric'].transform(X[numeric_cols])
        
        # Imputer categorical
        if 'categorical' in self.imputers_:
            cat_cols = [col for col in self.imputers_['categorical_cols'] if col in X.columns]
            if cat_cols:
                X[cat_cols] = self.imputers_['categorical'].transform(X[cat_cols].astype(str))
        
        return X


# ============================================================================
# CLASSE PRINCIPALE : DataPreprocessor (Version Production)
# ============================================================================

class DataPreprocessor:
    """
    Preprocessor de données ML avec fit/transform pattern
    Compatible avec base_model.py et data_loader.py
    
    Caractéristiques:
    - Pattern fit/transform pour éviter data leakage
    - Sauvegarde complète de tous les transformers
    - Validation robuste des données
    - Historique des transformations
    - Compatible avec sklearn Pipeline
    
    Exemple d'utilisation:
    >>> preprocessor = DataPreprocessor(verbose=True)
    >>> 
    >>> # Entraînement
    >>> preprocessor.fit(X_train, y_train)
    >>> X_train_processed = preprocessor.transform(X_train)
    >>> X_test_processed = preprocessor.transform(X_test)
    >>> 
    >>> # Sauvegarde
    >>> preprocessor.save('preprocessor.pkl')
    >>> 
    >>> # Production
    >>> preprocessor = DataPreprocessor.load('preprocessor.pkl')
    >>> X_new_processed = preprocessor.transform(X_new)
    """
    
    def __init__(self, 
                 problem_type: str = 'classification',
                 handle_missing: bool = True,
                 handle_outliers: bool = True,
                 encode_categorical: bool = True,
                 scale_features: bool = True,
                 select_features: bool = False,
                 balance_classes: bool = False,
                 verbose: bool = True):
        """
        Initialise le preprocessor
        
        Args:
            problem_type: 'classification' ou 'regression'
            handle_missing: Gérer les valeurs manquantes
            handle_outliers: Gérer les outliers
            encode_categorical: Encoder les variables catégorielles
            scale_features: Normaliser les features
            select_features: Sélectionner les meilleures features
            balance_classes: Équilibrer les classes (classification uniquement)
            verbose: Afficher les messages
        """
        self.problem_type = problem_type
        self.handle_missing = handle_missing
        self.handle_outliers = handle_outliers
        self.encode_categorical = encode_categorical
        self.scale_features = scale_features
        self.select_features = select_features
        self.balance_classes = balance_classes
        self.verbose = verbose
        
        # État
        self.is_fitted_ = False
        self.feature_names_in_ = []
        self.feature_names_out_ = []
        self.n_features_in_ = 0
        self.n_features_out_ = 0
        
        # Transformers
        self.transformers_ = {}
        
        # Métadonnées
        self.metadata_ = {
            'created_at': datetime.now().isoformat(),
            'fitted_at': None,
            'n_samples_seen': 0,
            'transformations_applied': []
        }
        
        # Configuration
        self.config_ = {
            'missing_threshold': 0.5,
            'outlier_method': 'clip',
            'outlier_strategy': 'iqr',
            'scaler_type': 'standard',
            'max_categories_onehot': 10,
            'feature_selection_k': 10,
            'balance_method': 'oversample'
        }
    
    def _log(self, message: str, level: str = 'INFO'):
        """Affiche un message si verbose"""
        if self.verbose:
            icons = {'INFO': '✓', 'WARNING': '⚠', 'ERROR': '✗', 'DEBUG': '→'}
            print(f"{icons.get(level, '•')} {message}")
    
    def set_config(self, **kwargs):
        """
        Configure les paramètres du preprocessor
        
        Args:
            missing_threshold: Seuil pour supprimer colonnes (0.5 = 50%)
            outlier_method: 'clip', 'remove'
            outlier_strategy: 'iqr', 'zscore'
            scaler_type: 'standard', 'minmax', 'robust'
            max_categories_onehot: Max catégories pour one-hot
            feature_selection_k: Nombre de features à garder
            balance_method: 'oversample', 'undersample', 'smote'
        """
        self.config_.update(kwargs)
        self._log(f"Configuration mise à jour: {kwargs}")
        return self
    
    def fit(self, X: pd.DataFrame, y: pd.Series = None) -> 'DataPreprocessor':
        """
        Apprend les transformations sur les données d'entraînement
        
        Args:
            X: Features d'entraînement
            y: Target (nécessaire pour sélection de features et équilibrage)
            
        Returns:
            self (pour chaining)
        """
        self._log("="*60)
        self._log("DÉMARRAGE DU FIT")
        self._log("="*60)
        
        X = X.copy()
        
        # Validation initiale
        self._validate_input_fit(X, y)
        
        # Sauvegarder info initiales
        self.feature_names_in_ = list(X.columns)
        self.n_features_in_ = len(X.columns)
        self.metadata_['n_samples_seen'] = len(X)
        
        # Pipeline de transformations
        step = 1
        
        # 1. Gestion des valeurs manquantes
        if self.handle_missing:
            self._log(f"\n{step}. Gestion des valeurs manquantes")
            imputer = SmartImputer(
                threshold_drop=self.config_['missing_threshold']
            )
            imputer.fit(X)
            self.transformers_['imputer'] = imputer
            X = imputer.transform(X)
            self.metadata_['transformations_applied'].append('imputation')
            self._log(f"   Colonnes après imputation: {len(X.columns)}")
            step += 1
        
        # 2. Encodage catégoriel
        if self.encode_categorical:
            self._log(f"\n{step}. Encodage des variables catégorielles")
            encoder = SmartEncoder(
                max_categories_onehot=self.config_['max_categories_onehot']
            )
            encoder.fit(X)
            self.transformers_['encoder'] = encoder
            X = encoder.transform(X)
            self.metadata_['transformations_applied'].append('encoding')
            self._log(f"   Colonnes après encodage: {len(X.columns)}")
            step += 1
        
        # 3. Gestion des outliers (sur données numériques)
        if self.handle_outliers:
            self._log(f"\n{step}. Gestion des outliers")
            outlier_handler = OutlierHandler(
                method=self.config_['outlier_method'],
                strategy=self.config_['outlier_strategy']
            )
            outlier_handler.fit(X)
            self.transformers_['outlier_handler'] = outlier_handler
            # Note: on n'applique pas transform ici pour ne pas modifier X
            # Ce sera fait dans transform()
            self.metadata_['transformations_applied'].append('outlier_handling')
            step += 1
        
        # 4. Sélection de features
        if self.select_features and y is not None:
            self._log(f"\n{step}. Sélection de features")
            score_func = f_classif if self.problem_type == 'classification' else f_regression
            k = min(self.config_['feature_selection_k'], len(X.columns))
            
            selector = SelectKBest(score_func=score_func, k=k)
            selector.fit(X, y)
            self.transformers_['selector'] = selector
            
            selected_features = X.columns[selector.get_support()].tolist()
            X = X[selected_features]
            
            self.metadata_['transformations_applied'].append('feature_selection')
            self._log(f"   Features sélectionnées: {len(selected_features)}")
            step += 1
        
        # 5. Scaling (toujours en dernier sur les features numériques)
        if self.scale_features:
            self._log(f"\n{step}. Normalisation des features")
            scaler_map = {
                'standard': StandardScaler(),
                'minmax': MinMaxScaler(),
                'robust': RobustScaler()
            }
            scaler = scaler_map.get(self.config_['scaler_type'], StandardScaler())
            
            numeric_cols = X.select_dtypes(include=[np.number]).columns
            if len(numeric_cols) > 0:
                scaler.fit(X[numeric_cols])
                self.transformers_['scaler'] = scaler
                self.transformers_['scaler_columns'] = list(numeric_cols)
                self.metadata_['transformations_applied'].append('scaling')
                self._log(f"   Scaler '{self.config_['scaler_type']}' appliqué")
            step += 1
        
        # Sauvegarder les noms de features finaux
        self.feature_names_out_ = list(X.columns)
        self.n_features_out_ = len(X.columns)
        
        # Marquer comme fitted
        self.is_fitted_ = True
        self.metadata_['fitted_at'] = datetime.now().isoformat()
        
        self._log("\n" + "="*60)
        self._log("FIT TERMINÉ")
        self._log(f"Features: {self.n_features_in_} → {self.n_features_out_}")
        self._log("="*60)
        
        return self
    
    def transform(self, X: pd.DataFrame) -> pd.DataFrame:
        """
        Applique les transformations apprises sur de nouvelles données
        
        Args:
            X: Données à transformer
            
        Returns:
            DataFrame transformé
        """
        if not self.is_fitted_:
            raise PreprocessorNotFittedError(
                "Le preprocessor doit être fitted avant transform. "
                "Appelez fit() d'abord."
            )
        
        self._validate_input_transform(X)
        
        X = X.copy()
        
        # Appliquer les transformations dans l'ordre
        if 'imputer' in self.transformers_:
            X = self.transformers_['imputer'].transform(X)
        
        if 'encoder' in self.transformers_:
            X = self.transformers_['encoder'].transform(X)
        
        if 'outlier_handler' in self.transformers_:
            X = self.transformers_['outlier_handler'].transform(X)
        
        if 'selector' in self.transformers_:
            selector = self.transformers_['selector']
            selected_cols = [col for col in X.columns if col in self.feature_names_out_]
            X = X[selected_cols]
        
        if 'scaler' in self.transformers_:
            scaler = self.transformers_['scaler']
            scaler_cols = [col for col in self.transformers_['scaler_columns'] 
                          if col in X.columns]
            if scaler_cols:
                X[scaler_cols] = scaler.transform(X[scaler_cols])
        
        # S'assurer que les colonnes correspondent exactement
        X = self._ensure_column_consistency(X)
        
        return X
    
    def fit_transform(self, X: pd.DataFrame, y: pd.Series = None) -> pd.DataFrame:
        """
        Fit puis transform en une seule étape
        
        Args:
            X: Features
            y: Target (optionnel)
            
        Returns:
            DataFrame transformé
        """
        return self.fit(X, y).transform(X)
    
    def _ensure_column_consistency(self, X: pd.DataFrame) -> pd.DataFrame:
        """
        S'assure que X a exactement les mêmes colonnes que lors du fit
        
        Args:
            X: DataFrame à vérifier
            
        Returns:
            DataFrame avec colonnes cohérentes
        """
        # Ajouter colonnes manquantes (avec 0)
        for col in self.feature_names_out_:
            if col not in X.columns:
                X[col] = 0
        
        # Supprimer colonnes en trop
        extra_cols = set(X.columns) - set(self.feature_names_out_)
        if extra_cols:
            X = X.drop(columns=list(extra_cols))
        
        # Réordonner dans le bon ordre
        return X[self.feature_names_out_]
    
    def _validate_input_fit(self, X: pd.DataFrame, y: pd.Series = None):
        """Valide les entrées pour fit"""
        if not isinstance(X, pd.DataFrame):
            raise TypeError("X doit être un pandas DataFrame")
        
        if X.empty:
            raise ValueError("X ne peut pas être vide")
        
        if self.select_features and y is None:
            raise ValueError("y est requis pour la sélection de features")
        
        if self.balance_classes and y is None:
            raise ValueError("y est requis pour l'équilibrage des classes")
        
        if y is not None:
            if not isinstance(y, pd.Series):
                raise TypeError("y doit être un pandas Series")
            if len(X) != len(y):
                raise ValueError("X et y doivent avoir la même longueur")
    
    def _validate_input_transform(self, X: pd.DataFrame):
        """Valide les entrées pour transform"""
        if not isinstance(X, pd.DataFrame):
            raise TypeError("X doit être un pandas DataFrame")
        
        if X.empty:
            raise ValueError("X ne peut pas être vide")
        
        # Vérifier que les colonnes essentielles sont présentes
        missing_cols = set(self.feature_names_in_) - set(X.columns)
        if missing_cols:
            self._log(f"Colonnes manquantes: {missing_cols}", 'WARNING')
            # On ne raise pas d'erreur car certaines colonnes peuvent avoir été
            # supprimées durant le fit (ex: trop de missing values)
    
    def balance_data(self, X: pd.DataFrame, y: pd.Series, 
                    method: str = None) -> Tuple[pd.DataFrame, pd.Series]:
        """
        Équilibre les classes déséquilibrées (classification uniquement)
        Cette méthode doit être appelée APRÈS fit et transform
        
        Args:
            X: Features transformées
            y: Target
            method: 'oversample', 'undersample', 'smote' (ou None pour config)
            
        Returns:
            Tuple (X_balanced, y_balanced)
        """
        if self.problem_type != 'classification':
            self._log("Équilibrage ignoré (pas une classification)", 'WARNING')
            return X, y
        
        method = method or self.config_['balance_method']
        
        class_counts = y.value_counts()
        self._log(f"Distribution avant: {class_counts.to_dict()}")
        
        if method == 'oversample':
            max_size = class_counts.max()
            dfs = []
            
            for class_label in class_counts.index:
                class_X = X[y == class_label]
                class_y = y[y == class_label]
                
                if len(class_X) < max_size:
                    class_X_resampled = resample(
                        class_X, 
                        replace=True,
                        n_samples=max_size,
                        random_state=42
                    )
                    class_y_resampled = pd.Series([class_label] * max_size)
                    dfs.append((class_X_resampled, class_y_resampled))
                else:
                    dfs.append((class_X, class_y))
            
            X_balanced = pd.concat([df[0] for df in dfs]).reset_index(drop=True)
            y_balanced = pd.concat([df[1] for df in dfs]).reset_index(drop=True)
            
        elif method == 'undersample':
            min_size = class_counts.min()
            dfs = []
            
            for class_label in class_counts.index:
                class_X = X[y == class_label]
                class_y = y[y == class_label]
                
                class_X_resampled = resample(
                    class_X,
                    replace=False,
                    n_samples=min_size,
                    random_state=42
                )
                class_y_resampled = pd.Series([class_label] * min_size)
                dfs.append((class_X_resampled, class_y_resampled))
            
            X_balanced = pd.concat([df[0] for df in dfs]).reset_index(drop=True)
            y_balanced = pd.concat([df[1] for df in dfs]).reset_index(drop=True)
            
        elif method == 'smote':
            try:
                from imblearn.over_sampling import SMOTE
                smote = SMOTE(random_state=42)
                X_balanced, y_balanced = smote.fit_resample(X, y)
                X_balanced = pd.DataFrame(X_balanced, columns=X.columns)
                y_balanced = pd.Series(y_balanced, name=y.name)
            except ImportError:
                self._log("SMOTE nécessite: pip install imbalanced-learn", 'ERROR')
                return X, y
        
        new_counts = y_balanced.value_counts()
        self._log(f"Distribution après: {new_counts.to_dict()}")
        
        return X_balanced, y_balanced
    
    def get_feature_names_out(self) -> List[str]:
        """Retourne les noms des features de sortie"""
        if not self.is_fitted_:
            raise PreprocessorNotFittedError("Preprocessor pas fitted")
        return self.feature_names_out_.copy()
    
    def get_preprocessing_report(self) -> Dict:
        """
        Génère un rapport détaillé du preprocessing
        
        Returns:
            Dictionnaire avec toutes les informations
        """
        return {
            'is_fitted': self.is_fitted_,
            'problem_type': self.problem_type,
            'n_features_in': self.n_features_in_,
            'n_features_out': self.n_features_out_,
            'feature_names_in': self.feature_names_in_,
            'feature_names_out': self.feature_names_out_,
            'transformers': list(self.transformers_.keys()),
            'config': self.config_,
            'metadata': self.metadata_
        }
    
    def save(self, filepath: str) -> bool:
        """
        Sauvegarde le preprocessor complet
        
        Args:
            filepath: Chemin de sauvegarde (.pkl)
            
        Returns:
            True si succès
        """
        try:
            filepath = Path(filepath)
            filepath.parent.mkdir(parents=True, exist_ok=True)
            
            with open(filepath, 'wb') as f:
                pickle.dump(self, f, protocol=pickle.HIGHEST_PROTOCOL)
            
            self._log(f"Preprocessor sauvegardé: {filepath}")
            return True
        except Exception as e:
            self._log(f"Erreur sauvegarde: {e}", 'ERROR')
            return False
    
    @classmethod
    def load(cls, filepath: str) -> 'DataPreprocessor':
        """
        Charge un preprocessor sauvegardé
        
        Args:
            filepath: Chemin du fichier
            
        Returns:
            DataPreprocessor chargé
        """
        try:
            with open(filepath, 'rb') as f:
                preprocessor = pickle.load(f)
            
            if not isinstance(preprocessor, cls):
                raise TypeError("Le fichier ne contient pas un DataPreprocessor")
            
            return preprocessor
        except Exception as e:
            raise IOError(f"Erreur chargement: {e}")


# ============================================================================
# FONCTION PRINCIPALE : AUTO_PREPROCESS_ML
# ============================================================================

def auto_preprocess_ml(df: pd.DataFrame, 
                       target_column: str,
                       problem_type: str = 'classification',
                       test_size: float = 0.2,
                       config: Dict = None,
                       verbose: bool = True) -> Dict:
    """
    Pipeline automatique complet de preprocessing ML (SANS DATA LEAKAGE)
    
    Cette fonction est le point d'entrée principal pour intégrer avec
    base_model.py et data_loader.py
    
    Args:
        df: DataFrame complet (avec target)
        target_column: Nom de la colonne cible
        problem_type: 'classification' ou 'regression'
        test_size: Proportion du test set (0.2 = 20%)
        config: Configuration personnalisée (optionnel)
        verbose: Afficher les messages
        
    Returns:
        Dictionnaire contenant:
        - X_train: Features d'entraînement transformées
        - X_test: Features de test transformées
        - y_train: Target d'entraînement
        - y_test: Target de test
        - preprocessor: DataPreprocessor fitted (pour transform ultérieur)
        - feature_names: Liste des noms de features
        - report: Rapport détaillé
        
    Exemple:
    >>> from data_loader import DataLoader
    >>> from models import create_model
    >>> 
    >>> # 1. Charger les données
    >>> loader = DataLoader()
    >>> df = loader.load_data("iris.csv")
    >>> 
    >>> # 2. Preprocessing automatique
    >>> result = auto_preprocess_ml(
    ...     df, 
    ...     target_column='species',
    ...     problem_type='classification'
    ... )
    >>> 
    >>> # 3. Entraîner un modèle
    >>> model = create_model('RandomForestClassifierModel', 'classification')
    >>> training_results = model.train(
    ...     result['X_train'], 
    ...     result['y_train'],
    ...     n_estimators=100
    ... )
    >>> 
    >>> # 4. Évaluer
    >>> eval_results = model.evaluate(result['X_test'], result['y_test'])
    >>> 
    >>> # 5. Production (nouvelles données)
    >>> X_new_processed = result['preprocessor'].transform(X_new)
    >>> predictions = model.predict(X_new_processed)
    """
    
    if verbose:
        print("\n" + "="*70)
        print("PIPELINE AUTOMATIQUE ML - SANS DATA LEAKAGE")
        print("="*70)
    
    df = df.copy()
    
    # Validation
    if target_column not in df.columns:
        raise ValueError(f"Colonne '{target_column}' introuvable dans le DataFrame")
    
    # 1. Séparation X et y
    if verbose:
        print("\n1️⃣  Séparation features / target")
    
    y = df[target_column]
    X = df.drop(columns=[target_column])
    
    if verbose:
        print(f"   Features: {X.shape[1]} colonnes")
        print(f"   Target: {target_column}")
        print(f"   Échantillons: {len(X)}")
    
    # 2. Split AVANT tout preprocessing (CRITIQUE pour éviter data leakage)
    if verbose:
        print(f"\n2️⃣  Split train/test ({int((1-test_size)*100)}% / {int(test_size*100)}%)")
    
    stratify = y if problem_type == 'classification' else None
    X_train, X_test, y_train, y_test = train_test_split(
        X, y, 
        test_size=test_size, 
        stratify=stratify, 
        random_state=42
    )
    
    if verbose:
        print(f"   Train: {len(X_train)} échantillons")
        print(f"   Test: {len(X_test)} échantillons")
    
    # 3. Créer et configurer le preprocessor
    if verbose:
        print("\n3️⃣  Configuration du preprocessor")
    
    preprocessor = DataPreprocessor(
        problem_type=problem_type,
        verbose=verbose
    )
    
    if config:
        preprocessor.set_config(**config)
    
    # 4. Fit sur TRAIN seulement
    if verbose:
        print("\n4️⃣  Fit du preprocessor (sur train uniquement)")
    
    preprocessor.fit(X_train, y_train)
    
    # 5. Transform train et test avec les MÊMES transformations
    if verbose:
        print("\n5️⃣  Transform train et test")
    
    X_train_processed = preprocessor.transform(X_train)
    X_test_processed = preprocessor.transform(X_test)
    
    if verbose:
        print(f"   Train transformé: {X_train_processed.shape}")
        print(f"   Test transformé: {X_test_processed.shape}")
    
    # 6. Équilibrage des classes (sur train seulement, si demandé)
    if preprocessor.balance_classes and problem_type == 'classification':
        if verbose:
            print("\n6️⃣  Équilibrage des classes (train uniquement)")
        
        X_train_processed, y_train = preprocessor.balance_data(
            X_train_processed, 
            y_train
        )
    
    # 7. Générer le rapport
    if verbose:
        print("\n" + "="*70)
        print("PREPROCESSING TERMINÉ")
        print("="*70)
    
    report = preprocessor.get_preprocessing_report()
    
    return {
        'X_train': X_train_processed,
        'X_test': X_test_processed,
        'y_train': y_train,
        'y_test': y_test,
        'preprocessor': preprocessor,
        'feature_names': preprocessor.get_feature_names_out(),
        'report': report,
        'summary': {
            'train_samples': len(X_train_processed),
            'test_samples': len(X_test_processed),
            'n_features': len(preprocessor.get_feature_names_out()),
            'problem_type': problem_type,
            'transformations': report['metadata']['transformations_applied']
        }
    }


# ============================================================================
# UTILITAIRES PRATIQUES
# ============================================================================

def quick_preprocess(X_train: pd.DataFrame, X_test: pd.DataFrame, 
                     y_train: pd.Series = None,
                     problem_type: str = 'classification') -> Dict:
    """
    Preprocessing rapide pour données déjà splitées
    
    Args:
        X_train: Features train
        X_test: Features test
        y_train: Target train (optionnel)
        problem_type: Type de problème
        
    Returns:
        Dict avec X_train et X_test transformés + preprocessor
    """
    preprocessor = DataPreprocessor(problem_type=problem_type, verbose=False)
    preprocessor.fit(X_train, y_train)
    
    return {
        'X_train': preprocessor.transform(X_train),
        'X_test': preprocessor.transform(X_test),
        'preprocessor': preprocessor
    }


def analyze_preprocessing_needs(df: pd.DataFrame, target_column: str = None) -> Dict:
    """
    Analyse les besoins de preprocessing d'un dataset
    
    Args:
        df: DataFrame à analyser
        target_column: Colonne cible (optionnel)
        
    Returns:
        Rapport d'analyse avec recommandations
    """
    if target_column and target_column in df.columns:
        X = df.drop(columns=[target_column])
    else:
        X = df.copy()
    
    report = {
        'shape': X.shape,
        'missing_values': {
            'total': int(X.isnull().sum().sum()),
            'by_column': X.isnull().sum().to_dict(),
            'percentage_by_column': (X.isnull().sum() / len(X) * 100).to_dict()
        },
        'duplicates': int(X.duplicated().sum()),
        'data_types': X.dtypes.astype(str).to_dict(),
        'categorical_columns': list(X.select_dtypes(include=['object', 'category']).columns),
        'numeric_columns': list(X.select_dtypes(include=[np.number]).columns),
        'recommendations': []
    }
    
    # Recommandations
    if report['missing_values']['total'] > 0:
        report['recommendations'].append(
            f"⚠️  {report['missing_values']['total']} valeurs manquantes détectées - "
            "Imputation recommandée"
        )
    
    if report['duplicates'] > 0:
        report['recommendations'].append(
            f"⚠️  {report['duplicates']} doublons détectés - Suppression recommandée"
        )
    
    if len(report['categorical_columns']) > 0:
        report['recommendations'].append(
            f"📊 {len(report['categorical_columns'])} colonnes catégorielles - "
            "Encodage nécessaire"
        )
    
    # Outliers
    numeric_cols = X.select_dtypes(include=[np.number]).columns
    if len(numeric_cols) > 0:
        outliers_detected = False
        for col in numeric_cols:
            Q1 = X[col].quantile(0.25)
            Q3 = X[col].quantile(0.75)
            IQR = Q3 - Q1
            outliers = ((X[col] < Q1 - 1.5*IQR) | (X[col] > Q3 + 1.5*IQR)).sum()
            if outliers > 0:
                outliers_detected = True
                break
        
        if outliers_detected:
            report['recommendations'].append(
                "⚠️  Outliers détectés - Traitement recommandé"
            )
    
    if len(report['recommendations']) == 0:
        report['recommendations'].append("✅ Données propres - Preprocessing minimal requis")
    
    return report