main.py

import streamlit as st
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
from sklearn.preprocessing import StandardScaler, MinMaxScaler, RobustScaler, LabelEncoder
from sklearn.impute import SimpleImputer, KNNImputer
from sklearn.ensemble import IsolationForest
from sklearn.model_selection import train_test_split
from sklearn.decomposition import PCA
from sklearn.manifold import TSNE
from sklearn.feature_selection import VarianceThreshold
from scipy import stats
from datetime import datetime
import tempfile
import os
import warnings
import plotly.express as px
import plotly.graph_objects as go
warnings.filterwarnings('ignore')

# Fixed CSS styling for better text visibility
def add_custom_css():
    st.markdown("""
    <style>
    /* Import modern font */
    @import url('https://fonts.googleapis.com/css2?family=Inter:wght@300;400;500;600;700&display=swap');
    
    /* Global background and text styling */
    .stApp {
        background-color: #000000 !important;
    }
    
    html, body, [class*="css"] {
        font-family: 'Inter', sans-serif !important;
        background-color: #000000 !important;
        color: #ffffff !important;
    }
    
    /* Main container styling */
    .main .block-container {
        padding-top: 2rem !important;
        padding-bottom: 2rem !important;
        max-width: 1200px !important;
        background-color: #000000 !important;
    }
    
    /* Text elements - white text on black background */
    .stMarkdown p, .stMarkdown li, .stMarkdown span {
        color: #ffffff !important;
        font-weight: 400 !important;
        line-height: 1.6 !important;
    }
    
    .stMarkdown h1, .stMarkdown h2, .stMarkdown h3, .stMarkdown h4 {
        color: #ffffff !important;
        font-weight: 600 !important;
        margin-bottom: 1rem !important;
    }
    
    /* Form elements styling */
    .stSelectbox label, .stCheckbox label, .stSlider label, .stTextInput label {
        color: #ffffff !important;
        font-weight: 500 !important;
        font-size: 0.95rem !important;
    }
    
    .stSelectbox > div > div {
        background-color: #1a1a1a !important;
        border: 1px solid #404040 !important;
        border-radius: 0.5rem !important;
        color: #ffffff !important;
    }
    
    /* Text input styling */
    .stTextInput > div > div > input {
        background-color: #1a1a1a !important;
        color: #ffffff !important;
        border: 1px solid #404040 !important;
    }
    
    /* Checkbox styling */
    .stCheckbox > label > div {
        background-color: #1a1a1a !important;
        border: 1px solid #404040 !important;
    }
    
    .stCheckbox > label {
        color: #ffffff !important;
    }
    
    /* Button styling */
    .stButton > button {
        background: linear-gradient(135deg, #3b82f6 0%, #1d4ed8 100%) !important;
        color: white !important;
        border: none !important;
        border-radius: 0.75rem !important;
        padding: 0.75rem 2rem !important;
        font-weight: 600 !important;
        font-size: 1rem !important;
        transition: all 0.2s ease !important;
        box-shadow: 0 4px 6px -1px rgba(0, 0, 0, 0.3) !important;
    }
    
    .stButton > button:hover {
        transform: translateY(-1px) !important;
        box-shadow: 0 10px 15px -3px rgba(0, 0, 0, 0.3) !important;
    }
    
    /* Expander styling */
    .streamlit-expanderHeader {
        background-color: #1a1a1a !important;
        border: 1px solid #404040 !important;
        border-radius: 0.5rem !important;
        color: #ffffff !important;
        font-weight: 600 !important;
    }
    
    .streamlit-expanderContent {
        background-color: #0a0a0a !important;
        border: 1px solid #404040 !important;
        border-top: none !important;
        border-radius: 0 0 0.5rem 0.5rem !important;
    }
    
    /* File uploader styling */
    .stFileUploader > label {
        color: #ffffff !important;
        font-weight: 500 !important;
    }
    
    .stFileUploader [data-testid="stFileUploaderDropzone"] {
        border: 2px dashed #666666 !important;
        border-radius: 0.75rem !important;
        background-color: #1a1a1a !important;
    }
    
    /* Dataframe styling */
    .stDataFrame {
        border: 1px solid #404040 !important;
        border-radius: 0.5rem !important;
        overflow: hidden !important;
    }
    
    /* Dataframe table styling */
    .stDataFrame table {
        background-color: #1a1a1a !important;
        color: #ffffff !important;
    }
    
    .stDataFrame th {
        background-color: #2a2a2a !important;
        color: #ffffff !important;
    }
    
    .stDataFrame td {
        background-color: #1a1a1a !important;
        color: #ffffff !important;
    }
    
    /* Metric styling */
    .metric-container {
        background-color: #1a1a1a !important;
        border: 1px solid #404040 !important;
        border-radius: 0.5rem !important;
        padding: 1rem !important;
        color: #ffffff !important;
    }
    
    /* Info/Warning/Error boxes */
    .stAlert {
        border-radius: 0.5rem !important;
    }
    
    .stInfo {
        background-color: #1a237e !important;
        border-left: 4px solid #3b82f6 !important;
        color: #90caf9 !important;
    }
    
    .stSuccess {
        background-color: #1b5e20 !important;
        border-left: 4px solid #22c55e !important;
        color: #a5d6a7 !important;
    }
    
    .stWarning {
        background-color: #e65100 !important;
        border-left: 4px solid #f59e0b !important;
        color: #ffcc02 !important;
    }
    
    .stError {
        background-color: #b71c1c !important;
        border-left: 4px solid #ef4444 !important;
        color: #ef9a9a !important;
    }
    
    /* Custom styled boxes */
    .info-box {
        background: linear-gradient(135deg, #667eea 0%, #764ba2 100%);
        color: white;
        padding: 1.5rem;
        border-radius: 1rem;
        margin: 1rem 0;
        box-shadow: 0 10px 25px rgba(0, 0, 0, 0.3);
    }
    
    .feature-box {
        background-color: #1a1a1a;
        border: 1px solid #404040;
        border-radius: 0.75rem;
        padding: 1rem;
        margin: 0.5rem 0;
        transition: all 0.2s ease;
        color: #ffffff;
    }
    
    .feature-box:hover {
        box-shadow: 0 4px 6px -1px rgba(255, 255, 255, 0.1);
        transform: translateY(-1px);
    }
    
    /* Sidebar styling */
    .css-1d391kg {
        background-color: #0a0a0a !important;
    }
    
    /* Sidebar content */
    .css-1lcbmhc {
        background-color: #0a0a0a !important;
    }
    
    /* Progress bar */
    .stProgress > div > div {
        background: linear-gradient(90deg, #3b82f6, #1d4ed8) !important;
    }
    
    /* Slider styling */
    .stSlider > div > div > div > div {
        background-color: #404040 !important;
    }
    
    /* Selectbox dropdown */
    .stSelectbox > div > div > div {
        background-color: #1a1a1a !important;
        color: #ffffff !important;
    }
    
    /* Remove default margins and improve spacing */
    .element-container {
        margin-bottom: 1rem !important;
    }
    
    /* Tab styling */
    .stTabs [data-baseweb="tab-list"] {
        background-color: #1a1a1a !important;
    }
    
    .stTabs [data-baseweb="tab"] {
        background-color: #1a1a1a !important;
        color: #ffffff !important;
    }
    
    .stTabs [aria-selected="true"] {
        background-color: #3b82f6 !important;
        color: #ffffff !important;
    }
    
    /* Responsive design */
    @media (max-width: 768px) {
        .main .block-container {
            padding-left: 1rem !important;
            padding-right: 1rem !important;
        }
    }
    </style>
    """, unsafe_allow_html=True)

class ComprehensiveDataPreprocessor:
    def __init__(self):
        self.report = []
        self.preprocessing_choices = {}
        self.fitted_transformers = {}
        self.data_info = {}
        
    def log_action(self, message):
        """Log preprocessing actions"""
        self.report.append(f"{datetime.now().strftime('%H:%M:%S')} - {message}")
        
    def set_preprocessing_choices(self, choices):
        """Set user choices for preprocessing steps"""
        self.preprocessing_choices = choices
    
    def load_and_inspect_data(self, file_path):
        """Load data and perform initial inspection"""
        try:
            if file_path.endswith('.csv'):
                df = pd.read_csv(file_path)
            elif file_path.endswith(('.xlsx', '.xls')):
                df = pd.read_excel(file_path)
            else:
                raise ValueError("Unsupported file format")
            
            self.data_info = {
                'original_shape': df.shape,
                'columns': list(df.columns),
                'dtypes': dict(df.dtypes),
                'missing_values': dict(df.isnull().sum()),
                'duplicates': df.duplicated().sum(),
                'memory_usage': df.memory_usage(deep=True).sum()
            }
            
            self.log_action(f"Loaded data: {df.shape[0]} rows, {df.shape[1]} columns")
            return df
            
        except Exception as e:
            self.log_action(f"Error loading data: {str(e)}")
            return None
    
    def remove_duplicates(self, df):
        """Remove exact duplicates"""
        if not self.preprocessing_choices.get('remove_duplicates', False):
            return df
        
        initial_count = len(df)
        df = df.drop_duplicates()
        removed = initial_count - len(df)
        
        if removed > 0:
            self.log_action(f"Removed {removed} duplicate rows")
        
        return df
    
    def handle_missing_values_advanced(self, df):
        """Handle missing values with different strategies"""
        if not self.preprocessing_choices.get('handle_missing', False):
            return df
        
        missing_strategy = self.preprocessing_choices.get('missing_strategy', 'auto')
        
        for col in df.columns:
            missing_pct = (df[col].isnull().sum() / len(df)) * 100
            
            if missing_pct == 0:
                continue
            
            if missing_pct > 80:
                df = df.drop(columns=[col])
                self.log_action(f"Dropped column '{col}' - {missing_pct:.1f}% missing")
                continue
            
            if df[col].dtype in ['int64', 'float64']:
                if missing_strategy == 'mean':
                    df[col].fillna(df[col].mean(), inplace=True)
                elif missing_strategy == 'median':
                    df[col].fillna(df[col].median(), inplace=True)
                elif missing_strategy == 'knn':
                    try:
                        imputer = KNNImputer(n_neighbors=5)
                        df[[col]] = imputer.fit_transform(df[[col]])
                    except:
                        df[col].fillna(df[col].median(), inplace=True)
                else:  # auto
                    try:
                        if abs(df[col].skew()) > 1:
                            df[col].fillna(df[col].median(), inplace=True)
                        else:
                            df[col].fillna(df[col].mean(), inplace=True)
                    except:
                        df[col].fillna(df[col].median(), inplace=True)
                
                self.log_action(f"Filled missing values in '{col}' using {missing_strategy}")
            
            elif df[col].dtype == 'object':
                try:
                    mode_val = df[col].mode()[0] if not df[col].mode().empty else 'Unknown'
                    df[col].fillna(mode_val, inplace=True)
                except:
                    df[col].fillna('Unknown', inplace=True)
                self.log_action(f"Filled missing categorical values in '{col}'")
        
        return df
    
    def fix_inconsistent_formats(self, df):
        """Fix inconsistent data formats"""
        if not self.preprocessing_choices.get('fix_formats', False):
            return df
        
        for col in df.columns:
            if any(keyword in col.lower() for keyword in ['date', 'time']):
                try:
                    df[col] = pd.to_datetime(df[col], errors='coerce')
                    self.log_action(f"Standardized date format in '{col}'")
                except:
                    pass
        
        return df
    
    def handle_outliers_advanced(self, df):
        """Handle outliers using different methods"""
        if not self.preprocessing_choices.get('handle_outliers', False):
            return df
        
        outlier_method = self.preprocessing_choices.get('outlier_method', 'iqr')
        numerical_cols = df.select_dtypes(include=[np.number]).columns
        
        for col in numerical_cols:
            try:
                if outlier_method == 'iqr':
                    Q1 = df[col].quantile(0.25)
                    Q3 = df[col].quantile(0.75)
                    IQR = Q3 - Q1
                    lower_bound = Q1 - 1.5 * IQR
                    upper_bound = Q3 + 1.5 * IQR
                    
                    outliers = ((df[col] < lower_bound) | (df[col] > upper_bound)).sum()
                    if outliers > 0:
                        df[col] = df[col].clip(lower=lower_bound, upper=upper_bound)
                        self.log_action(f"Capped {outliers} outliers in '{col}' using IQR method")
                
                elif outlier_method == 'zscore':
                    z_scores = np.abs(stats.zscore(df[col].dropna()))
                    outliers = (z_scores > 3).sum()
                    if outliers > 0:
                        median_val = df[col].median()
                        df.loc[df[col].index[z_scores > 3], col] = median_val
                        self.log_action(f"Replaced {outliers} outliers in '{col}' using Z-score method")
            except:
                continue
        
        return df
    
    def extract_datetime_features(self, df):
        """Extract features from datetime columns"""
        if not self.preprocessing_choices.get('datetime_features', False):
            return df
        
        datetime_cols = df.select_dtypes(include=['datetime64[ns]']).columns
        
        for col in datetime_cols:
            try:
                df[f'{col}_year'] = df[col].dt.year
                df[f'{col}_month'] = df[col].dt.month
                df[f'{col}_day'] = df[col].dt.day
                df[f'{col}_dayofweek'] = df[col].dt.dayofweek
                df[f'{col}_hour'] = df[col].dt.hour
                df[f'{col}_is_weekend'] = (df[col].dt.dayofweek >= 5).astype(int)
                
                self.log_action(f"Extracted datetime features from '{col}'")
            except:
                continue
        
        return df
    
    def extract_text_features(self, df):
        """Extract features from text columns"""
        if not self.preprocessing_choices.get('text_features', False):
            return df
        
        text_cols = df.select_dtypes(include=['object']).columns
        
        for col in text_cols[:3]:
            try:
                if df[col].nunique() / len(df) > 0.5:
                    df[f'{col}_length'] = df[col].astype(str).str.len()
                    df[f'{col}_word_count'] = df[col].astype(str).str.split().str.len()
                    
                    self.log_action(f"Extracted text features from '{col}'")
            except:
                continue
        
        return df
    
    def create_numeric_interactions(self, df):
        """Create interaction features between numeric variables"""
        if not self.preprocessing_choices.get('numeric_interactions', False):
            return df
        
        numerical_cols = df.select_dtypes(include=[np.number]).columns[:3]
        interaction_count = 0
        
        for i in range(len(numerical_cols)):
            for j in range(i+1, min(i+2, len(numerical_cols))):
                try:
                    col1, col2 = numerical_cols[i], numerical_cols[j]
                    df[f'{col1}_x_{col2}'] = df[col1] * df[col2]
                    interaction_count += 1
                except:
                    continue
        
        if interaction_count > 0:
            self.log_action(f"Created {interaction_count} interaction features")
        
        return df
    
    def apply_transformations(self, df):
        """Apply transformations like log, sqrt"""
        if not self.preprocessing_choices.get('apply_transformations', False):
            return df
        
        transformation_type = self.preprocessing_choices.get('transformation_type', 'log')
        numerical_cols = df.select_dtypes(include=[np.number]).columns
        
        for col in numerical_cols:
            try:
                if df[col].min() > 0 and df[col].skew() > 1:
                    if transformation_type == 'log':
                        df[f'{col}_log'] = np.log1p(df[col])
                        self.log_action(f"Applied log transformation to '{col}'")
                    elif transformation_type == 'sqrt':
                        df[f'{col}_sqrt'] = np.sqrt(df[col])
                        self.log_action(f"Applied sqrt transformation to '{col}'")
            except:
                continue
        
        return df
    
    def create_bins(self, df):
        """Create bins for continuous variables"""
        if not self.preprocessing_choices.get('create_bins', False):
            return df
        
        numerical_cols = df.select_dtypes(include=[np.number]).columns
        n_bins = self.preprocessing_choices.get('n_bins', 5)
        
        for col in numerical_cols[:2]:
            try:
                if df[col].nunique() > 10:
                    df[f'{col}_binned'] = pd.cut(df[col], bins=n_bins, labels=False)
                    self.log_action(f"Created {n_bins} bins for '{col}'")
            except:
                continue
        
        return df
    
    def encode_categorical_advanced(self, df):
        """Encode categorical variables"""
        if not self.preprocessing_choices.get('encode_categorical', False):
            return df
        
        categorical_cols = df.select_dtypes(include=['object', 'category']).columns
        encoding_method = self.preprocessing_choices.get('encoding_method', 'auto')
        
        for col in categorical_cols:
            try:
                unique_count = df[col].nunique()
                
                if encoding_method == 'auto':
                    if unique_count == 2:
                        le = LabelEncoder()
                        df[col] = le.fit_transform(df[col].astype(str))
                        self.log_action(f"Label encoded binary column '{col}'")
                    
                    elif unique_count <= 10:
                        dummies = pd.get_dummies(df[col], prefix=col, drop_first=True)
                        df = pd.concat([df, dummies], axis=1)
                        df = df.drop(columns=[col])
                        self.log_action(f"One-hot encoded '{col}'")
                    
                    else:
                        freq_encoding = df[col].value_counts().to_dict()
                        df[f'{col}_frequency'] = df[col].map(freq_encoding)
                        df = df.drop(columns=[col])
                        self.log_action(f"Frequency encoded '{col}'")
                
                elif encoding_method == 'label':
                    le = LabelEncoder()
                    df[col] = le.fit_transform(df[col].astype(str))
                    self.log_action(f"Label encoded '{col}'")
            except:
                continue
        
        return df
    
    def apply_scaling(self, df):
        """Apply scaling to numerical features"""
        if not self.preprocessing_choices.get('apply_scaling', False):
            return df
        
        scaling_method = self.preprocessing_choices.get('scaling_method', 'standard')
        numerical_cols = df.select_dtypes(include=[np.number]).columns
        
        # Exclude binary columns
        cols_to_scale = []
        for col in numerical_cols:
            try:
                unique_vals = df[col].unique()
                if not (len(unique_vals) == 2 and set(unique_vals).issubset({0, 1})):
                    cols_to_scale.append(col)
            except:
                continue
        
        if cols_to_scale:
            try:
                if scaling_method == 'standard':
                    scaler = StandardScaler()
                elif scaling_method == 'minmax':
                    scaler = MinMaxScaler()
                elif scaling_method == 'robust':
                    scaler = RobustScaler()
                
                df[cols_to_scale] = scaler.fit_transform(df[cols_to_scale])
                self.log_action(f"Applied {scaling_method} scaling to {len(cols_to_scale)} columns")
            except:
                self.log_action(f"Scaling failed, continuing without scaling")
        
        return df
    
    def feature_selection(self, df, target_column=None):
        """Perform feature selection"""
        if not self.preprocessing_choices.get('feature_selection', False):
            return df
        
        try:
            selection_method = self.preprocessing_choices.get('selection_method', 'variance')
            
            if selection_method == 'variance':
                selector = VarianceThreshold(threshold=0.01)
                numerical_cols = df.select_dtypes(include=[np.number]).columns
                
                if len(numerical_cols) > 0:
                    selected_features = selector.fit_transform(df[numerical_cols])
                    selected_cols = numerical_cols[selector.get_support()]
                    
                    other_cols = df.select_dtypes(exclude=[np.number]).columns
                    df_selected = pd.concat([
                        pd.DataFrame(selected_features, columns=selected_cols, index=df.index),
                        df[other_cols]
                    ], axis=1)
                    
                    removed_count = len(numerical_cols) - len(selected_cols)
                    self.log_action(f"Removed {removed_count} low-variance features")
                    return df_selected
        except:
            self.log_action("Feature selection failed, continuing with all features")
        
        return df
    
    def dimensionality_reduction(self, df, target_column=None):
        """Apply dimensionality reduction techniques"""
        if not self.preprocessing_choices.get('dimensionality_reduction', False):
            return df
        
        method = self.preprocessing_choices.get('reduction_method', 'pca')
        n_components = self.preprocessing_choices.get('n_components', 10)
        
        # Get numerical columns for reduction
        numerical_cols = df.select_dtypes(include=[np.number]).columns.tolist()
        if target_column and target_column in numerical_cols:
            numerical_cols.remove(target_column)
        
        if len(numerical_cols) < 2:
            self.log_action("Insufficient numerical columns for dimensionality reduction")
            return df
        
        # Ensure we don't reduce to more components than we have features
        n_components = min(n_components, len(numerical_cols))
        
        if len(numerical_cols) > n_components:
            try:
                # Prepare data for reduction
                X_for_reduction = df[numerical_cols].fillna(0)
                
                if method == 'pca':
                    reducer = PCA(n_components=n_components, random_state=42)
                    X_reduced = reducer.fit_transform(X_for_reduction)
                    
                    # Create new DataFrame with PCA components
                    pca_cols = [f'PCA_{i+1}' for i in range(n_components)]
                    df_reduced = pd.DataFrame(X_reduced, columns=pca_cols, index=df.index)
                    
                    self.fitted_transformers['pca_reducer'] = reducer
                    self.log_action(f"Applied PCA: {len(numerical_cols)} → {n_components} components")
                
                elif method == 'tsne':
                    # t-SNE parameters
                    perplexity = self.preprocessing_choices.get('tsne_perplexity', 30)
                    perplexity = min(perplexity, (len(df) - 1) // 3)
                    
                    # Limit data size for t-SNE performance
                    max_samples = min(1000, len(df))
                    if len(df) > max_samples:
                        sample_idx = np.random.choice(len(df), max_samples, replace=False)
                        X_sample = X_for_reduction.iloc[sample_idx]
                    else:
                        X_sample = X_for_reduction
                    
                    reducer = TSNE(n_components=min(n_components, 3), perplexity=perplexity, 
                                 random_state=42, max_iter=500)
                    X_reduced_sample = reducer.fit_transform(X_sample)
                    
                    # Create components for full dataset
                    tsne_cols = [f'tSNE_{i+1}' for i in range(X_reduced_sample.shape[1])]
                    
                    if len(df) > max_samples:
                        # For large datasets, fill with mean values for non-sampled data
                        mean_components = X_reduced_sample.mean(axis=0)
                        X_reduced = np.tile(mean_components, (len(df), 1))
                        X_reduced[sample_idx] = X_reduced_sample
                    else:
                        X_reduced = X_reduced_sample
                    
                    df_reduced = pd.DataFrame(X_reduced, columns=tsne_cols, index=df.index)
                    self.log_action(f"Applied t-SNE: {len(numerical_cols)} → {len(tsne_cols)} components")
                
                else:  # Fallback to PCA
                    reducer = PCA(n_components=n_components, random_state=42)
                    X_reduced = reducer.fit_transform(X_for_reduction)
                    pca_cols = [f'PCA_{i+1}' for i in range(n_components)]
                    df_reduced = pd.DataFrame(X_reduced, columns=pca_cols, index=df.index)
                    self.log_action(f"Applied PCA (fallback): {len(numerical_cols)} → {n_components} components")
                
                # Combine with non-numerical columns and target
                other_cols = df.select_dtypes(exclude=[np.number]).columns
                df_final = df_reduced.copy()
                
                # Add non-numerical columns
                if len(other_cols) > 0:
                    df_final = pd.concat([df_final, df[other_cols]], axis=1)
                
                # Add target column if specified
                if target_column and target_column in df.columns:
                    df_final[target_column] = df[target_column]
                
                return df_final
                
            except Exception as e:
                self.log_action(f"Dimensionality reduction failed: {str(e)}. Returning original data.")
                return df
        else:
            self.log_action(f"Dataset already has {len(numerical_cols)} features, no reduction needed")
            return df
    
    def generate_comprehensive_report(self, processed_df):
        """Generate comprehensive preprocessing report"""
        report = {
            'original_info': self.data_info,
            'processed_shape': processed_df.shape,
            'processing_steps': self.report,
            'fitted_transformers': list(self.fitted_transformers.keys()),
            'final_dtypes': dict(processed_df.dtypes),
            'final_missing_values': dict(processed_df.isnull().sum()),
            'preprocessing_choices': self.preprocessing_choices
        }
        
        return report
    
    def process_data(self, file_path, preprocessing_choices, target_column=None):
        """Main comprehensive preprocessing pipeline - returns entire dataset"""
        self.set_preprocessing_choices(preprocessing_choices)
        
        # Load data
        df = self.load_and_inspect_data(file_path)
        if df is None:
            return None, None
        
        try:
            # Apply preprocessing steps
            df = self.remove_duplicates(df)
            df = self.handle_missing_values_advanced(df)
            df = self.fix_inconsistent_formats(df)
            df = self.handle_outliers_advanced(df)
            df = self.extract_datetime_features(df)
            df = self.extract_text_features(df)
            df = self.create_numeric_interactions(df)
            df = self.apply_transformations(df)
            df = self.create_bins(df)
            df = self.encode_categorical_advanced(df)
            df = self.apply_scaling(df)
            df = self.feature_selection(df, target_column)
            df = self.dimensionality_reduction(df, target_column)
            
            # Generate report
            report = self.generate_comprehensive_report(df)
            
            self.log_action(f"Processing completed successfully! Final shape: {df.shape}")
            
            return df, report
            
        except Exception as e:
            self.log_action(f"Error during processing: {str(e)}")
            return None, None

def create_preprocessing_interface_with_explanations():
    """Create interface with detailed explanations for each preprocessing option"""
    
    # Apply custom CSS for better text visibility
    add_custom_css()
    
    st.title("🚀 Advanced Data Preprocessing App")
    
    # Enhanced introduction with improved design
    st.markdown("""
    <div class="info-box">
    <h2 style="color: white; margin-top: 0; font-weight: 700;">🎯 Transform Your Data Into ML-Ready Format</h2>
    <p style="font-size: 18px; margin-bottom: 15px; color: white; opacity: 0.95;">
    Automatically clean, engineer, and optimize your dataset with professional-grade preprocessing techniques. 
    Save hours of manual work and ensure your data is ready for machine learning.
    </p>
    
    <h3 style="color: white; margin-bottom: 15px;">✨ Key Features:</h3>
    <div style="display: grid; grid-template-columns: repeat(auto-fit, minmax(200px, 1fr)); gap: 15px; color: white;">
        <div>🤖 <strong>Smart Processing:</strong> Intelligent defaults</div>
        <div>🎛️ <strong>Full Control:</strong> Customize every step</div>
        <div>📊 <strong>Comprehensive:</strong> End-to-end pipeline</div>
        <div>⚡ <strong>Fast Results:</strong> Minutes not hours</div>
    </div>
    </div>
    """, unsafe_allow_html=True)
    
    # File upload section with improved styling
    st.markdown("### 📁 Upload Your Dataset")
    
    uploaded_file = st.file_uploader(
        "Choose your data file",
        type=['csv', 'xlsx', 'xls'],
        help="Supported formats: CSV, Excel (.xlsx, .xls). Maximum file size: 200MB"
    )
    
    if uploaded_file is None:
        st.info("👆 **Please upload a dataset to begin the preprocessing journey**")
        
        # Show example of what the app can do
        with st.expander("🎯 What can this app do for you?", expanded=True):
            st.markdown("""
            <div class="feature-box">
            <h4>🧹 Data Cleaning</h4>
            <p>• Remove duplicates and inconsistencies<br>
            • Handle missing values intelligently<br>
            • Fix date formats and data types</p>
            </div>
            
            <div class="feature-box">
            <h4>⚙️ Feature Engineering</h4>
            <p>• Extract datetime components<br>
            • Create text statistics<br>
            • Generate interaction features</p>
            </div>
            
            <div class="feature-box">
            <h4>🎯 ML Preparation</h4>
            <p>• Encode categorical variables<br>
            • Scale numerical features<br>
            • Reduce dimensionality with PCA/t-SNE</p>
            </div>
            """, unsafe_allow_html=True)
        
        return None
    
    # Initialize variables
    df = None
    target_column = None
    task_type = "exploration"
    
    # Load and preview data
    try:
        # Load data based on file type
        if uploaded_file.name.endswith('.csv'):
            df = pd.read_csv(uploaded_file)
        elif uploaded_file.name.endswith(('.xlsx', '.xls')):
            df = pd.read_excel(uploaded_file)
        else:
            st.error("❌ Unsupported file format. Please upload CSV or Excel files.")
            return None
        
        if df is None or df.empty:
            st.error("❌ The uploaded file is empty or couldn't be read.")
            return None
        
        # Success message
        st.success(f"✅ **File loaded successfully:** {df.shape[0]:,} rows × {df.shape[1]} columns")
        
        # Data preview
        with st.expander("📊 **Data Preview & Information**", expanded=False):
            col1, col2 = st.columns(2)
            
            with col1:
                st.markdown("**📋 First 5 Rows:**")
                st.dataframe(df.head(), use_container_width=True)
            
            with col2:
                st.markdown("**🔍 Data Summary:**")
                
                # Data info
                info_data = {
                    'Column': df.columns[:10],  # Show first 10 columns
                    'Type': [str(dtype) for dtype in df.dtypes[:10]],
                    'Missing': [df[col].isnull().sum() for col in df.columns[:10]],
                    'Unique': [df[col].nunique() for col in df.columns[:10]]
                }
                info_df = pd.DataFrame(info_data)
                st.dataframe(info_df, use_container_width=True)
                
                if len(df.columns) > 10:
                    st.info(f"Showing first 10 columns. Total columns: {len(df.columns)}")
        
        # Target selection and task type
        st.markdown("### 🎯 Configure Your Analysis")
        
        col1, col2 = st.columns(2)
        with col1:
            target_column = st.selectbox(
                "🎯 **Select Target Column** (Optional)",
                [None] + list(df.columns),
                help="Choose the variable you want to predict. Leave blank for unsupervised learning or data exploration."
            )
        
        with col2:
            if target_column:
                try:
                    unique_targets = df[target_column].nunique()
                    if unique_targets < 20:
                        task_type = st.selectbox(
                            "📊 **Task Type**", 
                            ["classification", "regression"],
                            help="**Classification:** Predicting categories (e.g., spam/not spam)\n**Regression:** Predicting continuous values (e.g., price, temperature)"
                        )
                    else:
                        task_type = "regression"
                        st.info("🔍 **Auto-detected:** Regression task (many unique values)")
                except Exception as e:
                    st.warning(f"⚠️ Could not analyze target column: {str(e)}")
                    task_type = st.selectbox(
                        "📊 **Task Type**", 
                        ["classification", "regression"],
                        help="Choose based on your prediction goal"
                    )
            else:
                task_type = st.selectbox(
                    "📊 **Analysis Type**", 
                    ["exploration", "classification", "regression"],
                    help="**Exploration:** Data analysis and insights\n**Classification/Regression:** Machine learning preparation"
                )
    
    except Exception as e:
        st.error(f"❌ **Error loading file:** {str(e)}")
        st.error("Please check if your file is valid and try again.")
        return None
    
    # Preprocessing Configuration
    st.markdown("### 🛠️ **Preprocessing Configuration**")
    st.markdown("Select the preprocessing steps you want to apply:")
    
    # Initialize variables with defaults
    preprocessing_vars = {}
    
    # Data Cleaning Section
    with st.expander("🧹 **Data Cleaning** - Essential quality improvements", expanded=True):
        st.markdown("""
        <div style="background: linear-gradient(135deg, #fef3c7 0%, #fde68a 100%); 
                    padding: 15px; border-radius: 8px; margin-bottom: 15px; color: #92400e;">
        <strong>🎯 Purpose:</strong> Clean and standardize your data for better quality and consistency.<br>
        <strong>💡 Recommendation:</strong> Always enable these steps for any dataset.
        </div>
        """, unsafe_allow_html=True)
        
        col1, col2 = st.columns(2)
        with col1:
            preprocessing_vars['remove_duplicates'] = st.checkbox(
                "🔄 **Remove Duplicate Rows**", 
                value=True,
                help="Removes identical rows that could skew your analysis"
            )
            
            preprocessing_vars['handle_missing'] = st.checkbox(
                "🕳️ **Handle Missing Values**", 
                value=True,
                help="Essential: Most ML algorithms can't handle missing data"
            )
        
        with col2:
            preprocessing_vars['fix_formats'] = st.checkbox(
                "📅 **Standardize Data Formats**", 
                value=True,
                help="Fixes dates, text cases, and ensures consistent data types"
            )
            
            preprocessing_vars['handle_outliers'] = st.checkbox(
                "📊 **Handle Outliers**", 
                value=True,
                help="Manages extreme values that could hurt model performance"
            )
        
        # Sub-options for data cleaning
        if preprocessing_vars.get('handle_missing', False):
            preprocessing_vars['missing_strategy'] = st.selectbox(
                "**Missing Value Strategy:**",
                ["auto", "mean", "median", "knn"],
                help="**Auto:** Smart choice based on data distribution\n**Mean/Median:** Simple statistical filling\n**KNN:** Advanced neighbor-based filling",
                index=0
            )
        else:
            preprocessing_vars['missing_strategy'] = "auto"
        
        if preprocessing_vars.get('handle_outliers', False):
            preprocessing_vars['outlier_method'] = st.selectbox(
                "**Outlier Handling Method:**",
                ["iqr", "zscore"],
                help="**IQR:** Most robust method (recommended)\n**Z-Score:** Statistical approach",
                index=0
            )
        else:
            preprocessing_vars['outlier_method'] = "iqr"
    
    # Feature Engineering Section
    with st.expander("⚙️ **Feature Engineering** - Create powerful new features"):
        st.markdown("""
        <div style="background: linear-gradient(135deg, #dbeafe 0%, #bfdbfe 100%); 
                    padding: 15px; border-radius: 8px; margin-bottom: 15px; color: #1e40af;">
        <strong>🎯 Purpose:</strong> Extract hidden patterns and create new meaningful variables.<br>
        <strong>💡 When to use:</strong> When you have datetime, text, or numeric data to enhance.
        </div>
        """, unsafe_allow_html=True)
        
        col1, col2 = st.columns(2)
        with col1:
            preprocessing_vars['datetime_features'] = st.checkbox(
                "📅 **Extract DateTime Features**",
                help="Creates year, month, day, weekday from date columns - great for time patterns"
            )
            
            preprocessing_vars['text_features'] = st.checkbox(
                "📝 **Extract Text Features**",
                help="Creates length, word count from text - useful for text analysis"
            )
        
        with col2:
            preprocessing_vars['numeric_interactions'] = st.checkbox(
                "🔢 **Create Numeric Interactions**",
                help="Multiplies numeric columns together - captures relationships"
            )
            
            preprocessing_vars['apply_transformations'] = st.checkbox(
                "📈 **Apply Math Transformations**",
                help="Log/sqrt transforms for skewed data - improves model performance"
            )
        
        # Sub-options for feature engineering
        if preprocessing_vars.get('apply_transformations', False):
            preprocessing_vars['transformation_type'] = st.selectbox(
                "**Transformation Type:**",
                ["log", "sqrt"],
                help="**Log:** Best for right-skewed data\n**Square Root:** Milder transformation"
            )
        else:
            preprocessing_vars['transformation_type'] = "log"
        
        preprocessing_vars['create_bins'] = st.checkbox(
            "📊 **Create Categorical Bins**",
            help="Converts continuous numbers into categories (e.g., age groups)"
        )
        
        if preprocessing_vars.get('create_bins', False):
            preprocessing_vars['n_bins'] = st.slider(
                "**Number of bins:**", 
                min_value=3, max_value=10, value=5,
                help="More bins = finer categories, fewer bins = broader groups"
            )
        else:
            preprocessing_vars['n_bins'] = 5
    
    # Categorical Encoding Section
    with st.expander("🏷️ **Categorical Encoding** - Convert text to numbers", expanded=True):
        st.markdown("""
        <div style="background: linear-gradient(135deg, #dcfce7 0%, #bbf7d0 100%); 
                    padding: 15px; border-radius: 8px; margin-bottom: 15px; color: #166534;">
        <strong>🎯 Purpose:</strong> Convert text categories to numbers for machine learning algorithms.<br>
        <strong>💡 Essential:</strong> Required for any ML model when you have categorical data.
        </div>
        """, unsafe_allow_html=True)
        
        preprocessing_vars['encode_categorical'] = st.checkbox(
            "🔤 **Encode Categorical Variables**", 
            value=True,
            help="Critical: ML algorithms need numbers, not text"
        )
        
        if preprocessing_vars.get('encode_categorical', False):
            preprocessing_vars['encoding_method'] = st.selectbox(
                "**Encoding Strategy:**",
                ["auto", "onehot", "label"],
                help="**Auto:** Smart choice based on data\n**One-Hot:** Creates binary columns (recommended)\n**Label:** Simple number assignment",
                index=0
            )
        else:
            preprocessing_vars['encoding_method'] = "auto"
    
    # Scaling Section
    with st.expander("📏 **Feature Scaling** - Normalize number ranges", expanded=True):
        st.markdown("""
        <div style="background: linear-gradient(135deg, #fce7f3 0%, #fbcfe8 100%); 
                    padding: 15px; border-radius: 8px; margin-bottom: 15px; color: #be185d;">
        <strong>🎯 Purpose:</strong> Put all features on the same scale so no feature dominates others.<br>
        <strong>💡 Critical for:</strong> Neural Networks, SVM, KNN, Clustering algorithms.
        </div>
        """, unsafe_allow_html=True)
        
        preprocessing_vars['apply_scaling'] = st.checkbox(
            "⚖️ **Apply Feature Scaling**", 
            value=True,
            help="Essential for distance-based algorithms and neural networks"
        )
        
        if preprocessing_vars.get('apply_scaling', False):
            preprocessing_vars['scaling_method'] = st.selectbox(
                "**Scaling Method:**",
                ["standard", "minmax", "robust"],
                help="**Standard:** Mean=0, Std=1 (most common)\n**MinMax:** Scales to 0-1 range\n**Robust:** Good with outliers",
                index=0
            )
        else:
            preprocessing_vars['scaling_method'] = "standard"
    
    # Advanced Processing Section
    with st.expander("🎯 **Advanced Optimization** - Maximum performance"):
        st.markdown("""
        <div style="background: linear-gradient(135deg, #f3e8ff 0%, #e9d5ff 100%); 
                    padding: 15px; border-radius: 8px; margin-bottom: 15px; color: #7c3aed;">
        <strong>🎯 Purpose:</strong> Advanced techniques for optimal model performance.<br>
        <strong>💡 Use when:</strong> You have many features or want maximum performance.
        </div>
        """, unsafe_allow_html=True)
        
        col1, col2 = st.columns(2)
        with col1:
            preprocessing_vars['feature_selection'] = st.checkbox(
                "🎯 **Feature Selection**",
                help="Removes irrelevant features - reduces overfitting"
            )
            
            if preprocessing_vars.get('feature_selection', False):
                preprocessing_vars['selection_method'] = st.selectbox(
                    "**Selection Method:**",
                    ["variance", "correlation"],
                    help="**Variance:** Removes low-variation features\n**Correlation:** Removes redundant features"
                )
            else:
                preprocessing_vars['selection_method'] = "variance"
        
        with col2:
            preprocessing_vars['dimensionality_reduction'] = st.checkbox(
                "📉 **Dimensionality Reduction**",
                help="Reduces features while preserving information - great for visualization"
            )
            
            if preprocessing_vars.get('dimensionality_reduction', False):
                preprocessing_vars['reduction_method'] = st.selectbox(
                    "**Reduction Method:**",
                    ["pca", "tsne"],
                    help="**PCA:** Linear, preserves variance (recommended)\n**t-SNE:** Non-linear, great for visualization"
                )
                
                preprocessing_vars['n_components'] = st.slider(
                    "**Number of components:**", 
                    min_value=2, max_value=min(50, len(df.columns)), value=10,
                    help="Fewer components = more compression, more components = more detail"
                )
                
                if preprocessing_vars.get('reduction_method') == "tsne":
                    preprocessing_vars['tsne_perplexity'] = st.slider(
                        "**t-SNE Perplexity:**", 
                        min_value=5, max_value=50, value=30,
                        help="Lower values emphasize local structure, higher values emphasize global structure"
                    )
                else:
                    preprocessing_vars['tsne_perplexity'] = 30
            else:
                preprocessing_vars['reduction_method'] = "pca"
                preprocessing_vars['n_components'] = 10
                preprocessing_vars['tsne_perplexity'] = 30
    
    # Processing Summary
    st.markdown("### 📋 **Processing Summary**")
    
    # Count enabled features and create summary
    enabled_features = []
    if preprocessing_vars.get('remove_duplicates'): enabled_features.append("✅ Remove Duplicates")
    if preprocessing_vars.get('handle_missing'): enabled_features.append("✅ Handle Missing Values")
    if preprocessing_vars.get('fix_formats'): enabled_features.append("✅ Fix Data Formats")
    if preprocessing_vars.get('handle_outliers'): enabled_features.append("✅ Handle Outliers")
    if preprocessing_vars.get('datetime_features'): enabled_features.append("✅ DateTime Features")
    if preprocessing_vars.get('text_features'): enabled_features.append("✅ Text Features")
    if preprocessing_vars.get('numeric_interactions'): enabled_features.append("✅ Numeric Interactions")
    if preprocessing_vars.get('apply_transformations'): enabled_features.append("✅ Math Transformations")
    if preprocessing_vars.get('create_bins'): enabled_features.append("✅ Create Bins")
    if preprocessing_vars.get('encode_categorical'): enabled_features.append("✅ Categorical Encoding")
    if preprocessing_vars.get('apply_scaling'): enabled_features.append("✅ Feature Scaling")
    if preprocessing_vars.get('feature_selection'): enabled_features.append("✅ Feature Selection")
    if preprocessing_vars.get('dimensionality_reduction'): enabled_features.append("✅ Dimensionality Reduction")
    
    if len(enabled_features) > 0:
        st.success(f"🎯 **{len(enabled_features)} preprocessing steps selected**")
        
        # Show in columns for better layout
        cols = st.columns(3)
        for i, feature in enumerate(enabled_features):
            cols[i % 3].write(feature)
    else:
        st.warning("⚠️ **No preprocessing steps selected.** Your data will be returned unchanged.")
    
    # Processing Button
    st.markdown("---")
    
    if len(enabled_features) > 0:
        if st.button("🚀 **Start Preprocessing**", type="primary", use_container_width=True):
            try:
                return {
                    'dataframe': df,
                    'choices': preprocessing_vars,
                    'target_column': target_column,
                    'task_type': task_type,
                    'uploaded_file': uploaded_file
                }
            except Exception as e:
                st.error(f"❌ **Error preparing configuration:** {str(e)}")
                return None
    else:
        st.button("🚀 **Start Preprocessing**", disabled=True, help="Please select at least one preprocessing step")
    
    return None

def display_processing_results(processed_df, report, original_result):
    """Display processing results with comprehensive information and improved styling"""
    
    st.markdown("---")
    st.markdown("## 🎉 **Processing Complete!**")
    
    # Results overview with metrics
    st.markdown("### 📊 **Transformation Summary**")
    
    col1, col2, col3, col4 = st.columns(4)
    with col1:
        st.metric(
            label="📥 Original Rows", 
            value=f"{report['original_info']['original_shape'][0]:,}",
            help="Number of rows in your original dataset"
        )
    with col2:
        processed_rows = report['processed_shape'][0]
        row_change = processed_rows - report['original_info']['original_shape'][0]
        st.metric(
            label="📤 Processed Rows", 
            value=f"{processed_rows:,}",
            delta=f"{row_change:+,}" if row_change != 0 else None,
            help="Number of rows after preprocessing"
        )
    with col3:
        st.metric(
            label="📋 Original Columns", 
            value=f"{report['original_info']['original_shape'][1]:,}",
            help="Number of columns in your original dataset"
        )
    with col4:
        processed_cols = report['processed_shape'][1]
        col_change = processed_cols - report['original_info']['original_shape'][1]
        st.metric(
            label="🎯 Processed Columns", 
            value=f"{processed_cols:,}",
            delta=f"{col_change:+,}" if col_change != 0 else None,
            help="Number of columns after preprocessing"
        )
    
    # Processing steps completed
    with st.expander("📋 **Processing Log** - See what was done", expanded=False):
        st.markdown("**Steps completed:**")
        for i, step in enumerate(report['processing_steps'], 1):
            st.write(f"{i}. {step}")
    
    # Dimensionality reduction visualization
    if original_result['choices'].get('dimensionality_reduction', False):
        st.markdown("### 📉 **Dimensionality Reduction Results**")
        
        method = original_result['choices'].get('reduction_method', 'pca')
        n_components = original_result['choices'].get('n_components', 10)
        
        st.info(f"🎯 Applied **{method.upper()}** reduction: {report['original_info']['original_shape'][1]} → {n_components} components")
        
        # Show component columns
        component_cols = [col for col in processed_df.columns if any(prefix in col for prefix in ['PCA_', 'tSNE_'])]
        if component_cols:
            st.write(f"**🔧 Generated Components:** {', '.join(component_cols[:10])}{'...' if len(component_cols) > 10 else ''}")
        
        # Display visualization if we have at least 2 components
        if len(component_cols) >= 2:
            try:
                fig = px.scatter(
                    processed_df, 
                    x=component_cols[0], 
                    y=component_cols[1],
                    title=f"{method.upper()} 2D Visualization",
                    opacity=0.7,
                    height=500
                )
                fig.update_layout(
                    plot_bgcolor='white',
                    paper_bgcolor='white',
                    font=dict(color='black')
                )
                st.plotly_chart(fig, use_container_width=True)
            except Exception as e:
                st.warning(f"Could not generate visualization: {str(e)}")
    
    # Data preview with better formatting
    st.markdown("### 🔍 **Processed Data Preview**")
    
    # Show data types summary
    col1, col2 = st.columns(2)
    with col1:
        st.markdown("**📊 Data Types:**")
        dtype_summary = processed_df.dtypes.value_counts()
        for dtype, count in dtype_summary.items():
            st.write(f"• **{dtype}:** {count} columns")
    
    with col2:
        st.markdown("**🔍 Quality Check:**")
        missing_count = processed_df.isnull().sum().sum()
        duplicate_count = processed_df.duplicated().sum()
        st.write(f"• **Missing values:** {missing_count:,}")
        st.write(f"• **Duplicate rows:** {duplicate_count:,}")
        st.write(f"• **Memory usage:** {processed_df.memory_usage(deep=True).sum() / 1024**2:.1f} MB")
    
    # Show actual data
    st.dataframe(
        processed_df.head(100), 
        use_container_width=True,
        height=400
    )
    
    if len(processed_df) > 100:
        st.info(f"📝 Showing first 100 rows. Total rows: {len(processed_df):,}")
    
    # Download section with enhanced styling
    st.markdown("### 📥 **Download Your Processed Dataset**")
    
    st.markdown("""
    <div style="background: #black; 
                padding: 20px; border-radius: 10px; margin: 15px 0; border-left: 5px solid #10b981;">
    <h4 style="color: #065f46; margin-top: 0;">🎯 Your data is now ML-ready!</h4>
    <p style="color: #047857; margin-bottom: 0;">
    The processed dataset includes all transformations and is optimized for machine learning algorithms.
    You can now use this data directly with scikit-learn, TensorFlow, PyTorch, or any other ML framework.
    </p>
    </div>
    """, unsafe_allow_html=True)
    
    # Create download button
    csv = processed_df.to_csv(index=False)
    timestamp = datetime.now().strftime('%Y%m%d_%H%M%S')
    
    st.download_button(
        label="📥 **Download Complete Processed Dataset (CSV)**",
        data=csv,
        file_name=f"processed_dataset_{timestamp}.csv",
        mime="text/csv",
        use_container_width=True,
        help="Download your preprocessed data ready for machine learning"
    )
    
    # Additional information
    with st.expander("💡 **Next Steps & Usage Tips**", expanded=False):
        st.markdown("""
        <div style="background-color: #black; padding: 15px; border-radius: 8px;">
        <h4 style="color: #1e293b;">🚀 What to do next:</h4>
        <ol style="color: #475569;">
            <li><strong>Download your data:</strong> Use the button above to get your processed dataset</li>
            <li><strong>Choose your ML algorithm:</strong> Your data is now compatible with any machine learning framework</li>
            <li><strong>Train your model:</strong> Split into train/test sets and start training</li>
            <li><strong>Evaluate performance:</strong> Use appropriate metrics for your task type</li>
        </ol>
        
        <h4 style="color: #1e293b;">📚 Recommended ML libraries:</h4>
        <ul style="color: #475569;">
            <li><strong>Scikit-learn:</strong> Great for beginners and classical ML</li>
            <li><strong>XGBoost/LightGBM:</strong> Excellent for tabular data competitions</li>
            <li><strong>TensorFlow/PyTorch:</strong> For deep learning applications</li>
        </ul>
        </div>
        """, unsafe_allow_html=True)

def main_app_with_explanations():
    """Main Streamlit app with enhanced UI and better text visibility"""
    
    st.set_page_config(
        page_title="Advanced Data Preprocessing App",
        page_icon="🚀",
        layout="wide",
        initial_sidebar_state="collapsed"
    )
    
    # Apply custom CSS
    add_custom_css()
    # Process the interface
    result = create_preprocessing_interface_with_explanations()
    
    if result:
        # Show processing progress
        progress_bar = st.progress(0)
        status_text = st.empty()
        
        with st.spinner("🔄 **Processing your data...** This may take a few moments."):
            try:
                # Initialize the preprocessor
                status_text.text("Initializing preprocessor...")
                progress_bar.progress(10)
                
                preprocessor = ComprehensiveDataPreprocessor()
                
                # Save uploaded file temporarily
                status_text.text("Preparing data...")
                progress_bar.progress(20)
                
                with tempfile.NamedTemporaryFile(delete=False, suffix='.csv') as tmp_file:
                    result['dataframe'].to_csv(tmp_file.name, index=False)
                    temp_file_path = tmp_file.name
                
                # Process the data
                status_text.text("Applying preprocessing steps...")
                progress_bar.progress(30)
                
                processed_df, report = preprocessor.process_data(
                    temp_file_path,
                    result['choices'],
                    result['target_column']
                )
                
                progress_bar.progress(90)
                status_text.text("Finalizing results...")
                
                # Clean up temp file
                try:
                    os.unlink(temp_file_path)
                except:
                    pass
                
                progress_bar.progress(100)
                status_text.text("Complete!")
                
                # Clear progress indicators
                progress_bar.empty()
                status_text.empty()
                
                if processed_df is not None:
                    st.balloons()  # Celebration animation
                    
                    # Display results
                    display_processing_results(processed_df, report, result)
                    
                    # Usage suggestions
                    st.markdown("### 🎯 **Suggested Next Steps**")
                    
                    task_type = result.get('task_type', 'exploration')
                    target_col = result.get('target_column')
                    
                    if task_type == "classification" and target_col:
                        st.info("""
                        **🎯 For Classification Tasks:**
                        - Try algorithms like Random Forest, XGBoost, or Logistic Regression
                        - Use metrics like accuracy, precision, recall, and F1-score
                        - Consider cross-validation for robust evaluation
                        """)
                    
                    elif task_type == "regression" and target_col:
                        st.info("""
                        **📈 For Regression Tasks:**
                        - Try algorithms like Random Forest, XGBoost, or Linear Regression
                        - Use metrics like MAE, MSE, RMSE, and R²
                        - Check for feature importance to understand your model
                        """)
                    
                    else:
                        st.info("""
                        **🔍 For Data Exploration:**
                        - Your data is now clean and ready for analysis
                        - Try clustering algorithms like K-Means for pattern discovery
                        - Use the processed features for visualization and insights
                        """)
                
                else:
                    st.error("❌ **Processing failed.** Please check your data and configuration, then try again.")
                    
                    # Show some debugging info
                    with st.expander("🔧 **Debugging Information**", expanded=False):
                        st.write("**Selected preprocessing steps:**")
                        for key, value in result['choices'].items():
                            if value and value != "auto":
                                st.write(f"• {key}: {value}")
                        
                        st.write("**Data info:**")
                        st.write(f"• Shape: {result['dataframe'].shape}")
                        st.write(f"• Columns: {list(result['dataframe'].columns)}")
                        st.write(f"• Target: {result['target_column']}")
                        
            except Exception as e:
                progress_bar.empty()
                status_text.empty()
                
                st.error(f"❌ **Error during processing:** {str(e)}")
                
                # Show detailed error information
                with st.expander("🔧 **Error Details**", expanded=False):
                    st.code(str(e))
                    st.write("**Possible solutions:**")
                    st.write("• Check if your data file is valid and not corrupted")
                    st.write("• Try with fewer preprocessing steps enabled")
                    st.write("• Ensure your dataset is not too large (>200MB)")
                    st.write("• Check if there are any special characters in column names")
                
                st.info("💡 **Tip:** Try enabling fewer preprocessing steps or check your data format.")

if __name__ == "__main__":
    main_app_with_explanations()