"""
SAMMS Enterprise - AI & Analytics Engine Schemas
AI-powered analytics, forecasting, anomaly detection, and decision support
"""

from datetime import datetime, date
from typing import Optional, List, Dict, Any, Union, Literal
from decimal import Decimal
from enum import Enum
from pydantic import BaseModel, Field, ConfigDict, field_validator, model_validator, computed_field
from .base import (
    BaseResponseSchema,
    TimestampMixin,
    PaginationParams,
    ListResponseBase,
    AuditMixin,
    CompanyScopedMixin
)


# ==================== ENUMS ====================

class AIModelType(str, Enum):
    """Types of AI/ML models"""
    FORECASTING = "forecasting"
    CLASSIFICATION = "classification"
    REGRESSION = "regression"
    CLUSTERING = "clustering"
    ANOMALY_DETECTION = "anomaly_detection"
    NLP = "nlp"
    RECOMMENDATION = "recommendation"
    OPTIMIZATION = "optimization"
    TIME_SERIES = "time_series"
    SENTIMENT_ANALYSIS = "sentiment_analysis"
    ENTITY_EXTRACTION = "entity_extraction"
    DOCUMENT_ANALYSIS = "document_analysis"


class ModelStatus(str, Enum):
    """AI model lifecycle status"""
    DRAFT = "draft"
    TRAINING = "training"
    VALIDATING = "validating"
    ACTIVE = "active"
    DEPRECATED = "deprecated"
    ARCHIVED = "archived"
    FAILED = "failed"


class AnomalyType(str, Enum):
    """Types of anomalies detected"""
    OUTLIER = "outlier"
    TREND_CHANGE = "trend_change"
    THRESHOLD_BREACH = "threshold_breach"
    PATTERN_DEVIATION = "pattern_deviation"
    SEASONAL_ANOMALY = "seasonal_anomaly"
    TRANSACTION_ANOMALY = "transaction_anomaly"
    BEHAVIORAL_ANOMALY = "behavioral_anomaly"
    COMPLIANCE_ANOMALY = "compliance_anomaly"


class AnomalySeverity(str, Enum):
    """Severity levels for anomalies"""
    LOW = "low"
    MEDIUM = "medium"
    HIGH = "high"
    CRITICAL = "critical"


class ForecastType(str, Enum):
    """Types of forecasts"""
    CASH_FLOW = "cash_flow"
    REVENUE = "revenue"
    EXPENSE = "expense"
    DEMAND = "demand"
    HEADCOUNT = "headcount"
    INVENTORY = "inventory"
    SALES = "sales"
    PROFIT = "profit"
    BUDGET = "budget"
    CUSTOM = "custom"


class ForecastHorizon(str, Enum):
    """Forecast time horizons"""
    DAILY = "daily"
    WEEKLY = "weekly"
    MONTHLY = "monthly"
    QUARTERLY = "quarterly"
    YEARLY = "yearly"


class AlertType(str, Enum):
    """Types of AI-generated alerts"""
    THRESHOLD = "threshold"
    ANOMALY = "anomaly"
    FORECAST_DEVIATION = "forecast_deviation"
    TREND = "trend"
    COMPLIANCE = "compliance"
    RISK = "risk"
    OPPORTUNITY = "opportunity"
    RECOMMENDATION = "recommendation"


class AlertSeverity(str, Enum):
    """Alert severity levels"""
    INFO = "info"
    WARNING = "warning"
    ERROR = "error"
    CRITICAL = "critical"


class InsightType(str, Enum):
    """Types of AI insights"""
    TREND = "trend"
    PATTERN = "pattern"
    CORRELATION = "correlation"
    OUTLIER = "outlier"
    SEGMENT = "segment"
    PREDICTION = "prediction"
    RECOMMENDATION = "recommendation"
    EXPLANATION = "explanation"
    SUMMARY = "summary"
    COMPARISON = "comparison"


class RecommendationType(str, Enum):
    """Types of recommendations"""
    COST_SAVING = "cost_saving"
    REVENUE_OPTIMIZATION = "revenue_optimization"
    RISK_MITIGATION = "risk_mitigation"
    EFFICIENCY_IMPROVEMENT = "efficiency_improvement"
    COMPLIANCE_ACTION = "compliance_action"
    INVESTMENT = "investment"
    RESOURCE_ALLOCATION = "resource_allocation"
    PROCESS_IMPROVEMENT = "process_improvement"
    CASH_MANAGEMENT = "cash_management"
    VENDOR_OPTIMIZATION = "vendor_optimization"


class RecommendationPriority(str, Enum):
    """Recommendation priority levels"""
    LOW = "low"
    MEDIUM = "medium"
    HIGH = "high"
    URGENT = "urgent"


class AnalysisType(str, Enum):
    """Types of document/contract analysis"""
    CONTRACT_REVIEW = "contract_review"
    RISK_ASSESSMENT = "risk_assessment"
    COMPLIANCE_CHECK = "compliance_check"
    TERM_EXTRACTION = "term_extraction"
    SENTIMENT_ANALYSIS = "sentiment_analysis"
    SIMILARITY_CHECK = "similarity_check"
    KEY_CLAUSE_EXTRACTION = "key_clause_extraction"
    OBLIGATION_EXTRACTION = "obligation_extraction"


class DataSource(str, Enum):
    """Data sources for AI models"""
    GL_TRANSACTIONS = "gl_transactions"
    JOURNAL_ENTRIES = "journal_entries"
    INVOICES = "invoices"
    PAYMENTS = "payments"
    TREASURY = "treasury"
    PAYROLL = "payroll"
    BUDGETS = "budgets"
    CONTRACTS = "contracts"
    DOCUMENTS = "documents"
    TIME_SERIES = "time_series"
    EXTERNAL = "external"


# ==================== AI MODEL ====================

class AIModelConfigBase(BaseModel):
    """Base schema for AI model configuration"""
    model_name: str = Field(..., min_length=1, max_length=200, description="Model name")
    model_version: str = Field(default="1.0.0", description="Model version")
    model_type: AIModelType = Field(..., description="Type of AI model")
    algorithm: str = Field(..., min_length=1, max_length=100, description="Algorithm used")
    framework: str = Field(default="sklearn", description="ML framework: sklearn, tensorflow, pytorch, etc.")
    
    # Training configuration
    features: List[str] = Field(..., min_length=1, description="Feature columns used")
    target_variable: Optional[str] = Field(None, description="Target variable for supervised learning")
    hyperparameters: Optional[Dict[str, Any]] = Field(None, description="Model hyperparameters")
    preprocessing_config: Optional[Dict[str, Any]] = Field(None, description="Data preprocessing configuration")
    
    # Training data
    training_data_source: DataSource = Field(..., description="Training data source")
    training_data_query: Optional[str] = Field(None, description="SQL query or data filter")
    training_date_range: Optional[Dict[str, date]] = Field(None, description="Training data date range")
    validation_split: Decimal = Field(default=Decimal('0.2'), ge=0, le=0.5, description="Validation split ratio")
    test_split: Decimal = Field(default=Decimal('0.1'), ge=0, le=0.3, description="Test split ratio")
    
    # Training execution
    auto_retrain: bool = Field(default=False, description="Enable automatic retraining")
    retrain_frequency: Optional[str] = Field(None, description="Retraining frequency")
    min_samples_retrain: Optional[int] = Field(None, ge=100, description="Minimum new samples for retrain")
    last_trained_at: Optional[datetime] = Field(None, description="Last training timestamp")
    training_duration_seconds: Optional[int] = Field(None, description="Training duration")
    
    # Performance thresholds
    min_accuracy: Optional[Decimal] = Field(None, ge=0, le=1, description="Minimum accuracy threshold")
    min_precision: Optional[Decimal] = Field(None, ge=0, le=1, description="Minimum precision threshold")
    min_recall: Optional[Decimal] = Field(None, ge=0, le=1, description="Minimum recall threshold")
    
    # Deployment
    is_active: bool = Field(default=False, description="Whether model is active")
    deployment_endpoint: Optional[str] = Field(None, max_length=500, description="Model API endpoint")
    batch_size: Optional[int] = Field(None, ge=1, description="Batch size for predictions")


class AIModelConfigCreate(AIModelConfigBase):
    """Schema for creating AI model configurations"""
    pass


class AIModelConfigUpdate(BaseModel):
    """Schema for updating AI model configurations"""
    model_name: Optional[str] = Field(None, min_length=1, max_length=200)
    hyperparameters: Optional[Dict[str, Any]] = None
    auto_retrain: Optional[bool] = None
    retrain_frequency: Optional[str] = None
    min_accuracy: Optional[Decimal] = Field(None, ge=0, le=1)
    min_precision: Optional[Decimal] = Field(None, ge=0, le=1)
    min_recall: Optional[Decimal] = Field(None, ge=0, le=1)
    is_active: Optional[bool] = None
    deployment_endpoint: Optional[str] = Field(None, max_length=500)


class ModelMetrics(BaseModel):
    """Model performance metrics"""
    accuracy: Optional[Decimal] = Field(None, description="Overall accuracy")
    precision: Optional[Decimal] = Field(None, description="Precision score")
    recall: Optional[Decimal] = Field(None, description="Recall score")
    f1_score: Optional[Decimal] = Field(None, description="F1 score")
    auc_roc: Optional[Decimal] = Field(None, description="AUC-ROC score")
    mae: Optional[Decimal] = Field(None, description="Mean Absolute Error")
    mse: Optional[Decimal] = Field(None, description="Mean Squared Error")
    rmse: Optional[Decimal] = Field(None, description="Root Mean Squared Error")
    mape: Optional[Decimal] = Field(None, description="Mean Absolute Percentage Error")
    r_squared: Optional[Decimal] = Field(None, description="R-squared score")
    custom_metrics: Optional[Dict[str, Decimal]] = Field(None, description="Custom metrics")


class FeatureImportance(BaseModel):
    """Feature importance values"""
    feature_name: str = Field(..., description="Feature name")
    importance_score: Decimal = Field(..., description="Importance score")
    rank: int = Field(..., ge=1, description="Importance rank")


class AIModelConfigResponse(AIModelConfigBase, BaseResponseSchema, CompanyScopedMixin):
    """Schema for AI model configuration responses"""
    status: ModelStatus = Field(default=ModelStatus.DRAFT)
    metrics: Optional[ModelMetrics] = Field(None, description="Model performance metrics")
    feature_importance: Optional[List[FeatureImportance]] = Field(None, description="Feature importance rankings")
    training_samples: int = Field(default=0, description="Number of training samples")
    validation_samples: int = Field(default=0, description="Number of validation samples")
    prediction_count: int = Field(default=0, description="Total predictions made")
    last_prediction_at: Optional[datetime] = Field(None, description="Last prediction timestamp")
    created_by_id: int

    model_config = ConfigDict(from_attributes=True)


# ==================== ANOMALY DETECTION ====================

class AnomalyDetectionConfigBase(BaseModel):
    """Base schema for anomaly detection configuration"""
    name: str = Field(..., min_length=1, max_length=200, description="Configuration name")
    description: Optional[str] = Field(None, max_length=2000, description="Description")
    anomaly_type: AnomalyType = Field(..., description="Type of anomaly to detect")
    data_source: DataSource = Field(..., description="Data source to monitor")
    data_query: Optional[str] = Field(None, description="Data query/filter")
    
    # Detection parameters
    detection_method: str = Field(default="statistical", description="Detection method: statistical, ml, rules, hybrid")
    sensitivity: Decimal = Field(default=Decimal('0.95'), ge=0.5, le=0.99, description="Detection sensitivity")
    threshold_type: str = Field(default="dynamic", description="Threshold type: static, dynamic, adaptive")
    static_threshold_low: Optional[Decimal] = Field(None, description="Static lower threshold")
    static_threshold_high: Optional[Decimal] = Field(None, description="Static upper threshold")
    
    # Time window
    lookback_period: int = Field(default=30, ge=1, le=365, description="Lookback period in days")
    comparison_period: int = Field(default=7, ge=1, le=90, description="Comparison period in days")
    seasonality_period: Optional[int] = Field(None, description="Seasonality period in days")
    
    # ML configuration
    model_id: Optional[int] = Field(None, description="Linked ML model ID")
    
    # Alerting
    generate_alerts: bool = Field(default=True, description="Generate alerts for detected anomalies")
    alert_severity_threshold: AnomalySeverity = Field(default=AnomalySeverity.MEDIUM, description="Minimum severity for alerts")
    notification_channels: Optional[List[str]] = Field(None, description="Notification channels")
    
    # Execution
    is_active: bool = Field(default=True, description="Whether detection is active")
    scan_frequency: str = Field(default="daily", description="Scan frequency: hourly, daily, weekly")
    last_scan_at: Optional[datetime] = Field(None, description="Last scan timestamp")
    next_scan_at: Optional[datetime] = Field(None, description="Next scheduled scan")


class AnomalyDetectionConfigCreate(AnomalyDetectionConfigBase):
    """Schema for creating anomaly detection configurations"""
    pass


class AnomalyDetectionConfigUpdate(BaseModel):
    """Schema for updating anomaly detection configurations"""
    name: Optional[str] = Field(None, min_length=1, max_length=200)
    description: Optional[str] = Field(None, max_length=2000)
    sensitivity: Optional[Decimal] = Field(None, ge=0.5, le=0.99)
    threshold_type: Optional[str] = None
    static_threshold_low: Optional[Decimal] = None
    static_threshold_high: Optional[Decimal] = None
    lookback_period: Optional[int] = Field(None, ge=1, le=365)
    comparison_period: Optional[int] = Field(None, ge=1, le=90)
    generate_alerts: Optional[bool] = None
    alert_severity_threshold: Optional[AnomalySeverity] = None
    is_active: Optional[bool] = None
    scan_frequency: Optional[str] = None


class AnomalyDetectionConfigResponse(AnomalyDetectionConfigBase, BaseResponseSchema, CompanyScopedMixin):
    """Schema for anomaly detection configuration responses"""
    total_anomalies_detected: int = Field(default=0, description="Total anomalies detected")
    anomalies_last_30_days: int = Field(default=0, description="Anomalies in last 30 days")
    false_positive_rate: Optional[Decimal] = Field(None, description="Estimated false positive rate")
    model_config = ConfigDict(from_attributes=True)


class DetectedAnomalyBase(BaseModel):
    """Base schema for detected anomalies"""
    detection_config_id: int = Field(..., description="Detection configuration ID")
    anomaly_type: AnomalyType = Field(..., description="Type of anomaly")
    severity: AnomalySeverity = Field(..., description="Anomaly severity")
    detected_at: datetime = Field(..., description="Detection timestamp")
    
    # Anomaly details
    entity_type: str = Field(..., description="Affected entity type: transaction, account, vendor, etc.")
    entity_id: Optional[int] = Field(None, description="Affected entity ID")
    entity_identifier: Optional[str] = Field(None, description="Entity identifier (e.g., transaction number)")
    
    # Values
    expected_value: Optional[Decimal] = Field(None, description="Expected value")
    actual_value: Optional[Decimal] = Field(None, description="Actual value")
    deviation: Optional[Decimal] = Field(None, description="Deviation from expected")
    deviation_percentage: Optional[Decimal] = Field(None, description="Deviation percentage")
    z_score: Optional[Decimal] = Field(None, description="Z-score of the anomaly")
    confidence_score: Decimal = Field(..., ge=0, le=1, description="Detection confidence score")
    
    # Context
    detection_date: date = Field(..., description="Date of anomalous data")
    period_context: Optional[Dict[str, Any]] = Field(None, description="Period context data")
    related_entities: Optional[List[Dict[str, Any]]] = Field(None, description="Related entities")
    
    # Status
    status: str = Field(default="new", description="Status: new, investigating, confirmed, false_positive, resolved")
    assigned_to_id: Optional[int] = Field(None, description="Assigned user ID")
    assigned_at: Optional[datetime] = Field(None, description="Assignment timestamp")
    
    # Resolution
    resolution_notes: Optional[str] = Field(None, max_length=5000, description="Resolution notes")
    resolved_by_id: Optional[int] = Field(None, description="User who resolved")
    resolved_at: Optional[datetime] = Field(None, description="Resolution timestamp")
    resolution_type: Optional[str] = Field(None, description="Resolution type: action_taken, false_positive, acceptable, other")
    
    # Additional context
    description: str = Field(..., min_length=1, max_length=2000, description="Anomaly description")
    supporting_data: Optional[Dict[str, Any]] = Field(None, description="Supporting data points")


class DetectedAnomalyCreate(DetectedAnomalyBase):
    """Schema for creating detected anomaly records"""
    pass


class DetectedAnomalyUpdate(BaseModel):
    """Schema for updating detected anomalies"""
    status: Optional[str] = None
    assigned_to_id: Optional[int] = None
    resolution_notes: Optional[str] = Field(None, max_length=5000)
    resolution_type: Optional[str] = None


class DetectedAnomalyResponse(DetectedAnomalyBase, BaseResponseSchema, CompanyScopedMixin):
    """Schema for detected anomaly responses"""
    detection_config_name: Optional[str] = None
    assigned_to_name: Optional[str] = None
    resolved_by_name: Optional[str] = None
    days_open: int = Field(default=0, description="Days since detection")
    model_config = ConfigDict(from_attributes=True)


# ==================== FORECASTING ====================

class ForecastConfigBase(BaseModel):
    """Base schema for forecast configuration"""
    name: str = Field(..., min_length=1, max_length=200, description="Forecast name")
    description: Optional[str] = Field(None, max_length=2000, description="Description")
    forecast_type: ForecastType = Field(..., description="Type of forecast")
    forecast_horizon: ForecastHorizon = Field(default=ForecastHorizon.MONTHLY, description="Forecast horizon granularity")
    horizon_periods: int = Field(default=12, ge=1, le=60, description="Number of periods to forecast")
    
    # Data source
    data_source: DataSource = Field(..., description="Data source for forecasting")
    data_query: Optional[str] = Field(None, description="Data query/filter")
    historical_periods: int = Field(default=24, ge=6, le=120, description="Historical periods for training")
    
    # Forecasting method
    method: str = Field(default="auto", description="Forecasting method: auto, arima, exponential_smoothing, prophet, ml, ensemble")
    model_id: Optional[int] = Field(None, description="Linked ML model ID")
    
    # Parameters
    include_seasonality: bool = Field(default=True, description="Include seasonality")
    seasonality_type: Optional[str] = Field(None, description="Seasonality type: additive, multiplicative")
    include_trend: bool = Field(default=True, description="Include trend component")
    confidence_intervals: List[Decimal] = Field(default=[Decimal('0.80'), Decimal('0.95')], description="Confidence intervals")
    
    # External factors
    external_factors: Optional[List[str]] = Field(None, description="External factor columns")
    scenario_adjustments: Optional[Dict[str, Decimal]] = Field(None, description="Scenario adjustments")
    
    # Execution
    is_active: bool = Field(default=True, description="Whether forecast is active")
    auto_refresh: bool = Field(default=True, description="Auto-refresh forecasts")
    refresh_frequency: str = Field(default="monthly", description="Refresh frequency")
    last_forecast_at: Optional[datetime] = Field(None, description="Last forecast timestamp")
    next_forecast_at: Optional[datetime] = Field(None, description="Next scheduled forecast")


class ForecastConfigCreate(ForecastConfigBase):
    """Schema for creating forecast configurations"""
    pass


class ForecastConfigUpdate(BaseModel):
    """Schema for updating forecast configurations"""
    name: Optional[str] = Field(None, min_length=1, max_length=200)
    description: Optional[str] = Field(None, max_length=2000)
    horizon_periods: Optional[int] = Field(None, ge=1, le=60)
    historical_periods: Optional[int] = Field(None, ge=6, le=120)
    method: Optional[str] = None
    include_seasonality: Optional[bool] = None
    include_trend: Optional[bool] = None
    confidence_intervals: Optional[List[Decimal]] = None
    external_factors: Optional[List[str]] = None
    scenario_adjustments: Optional[Dict[str, Decimal]] = None
    is_active: Optional[bool] = None
    auto_refresh: Optional[bool] = None
    refresh_frequency: Optional[str] = None


class ForecastConfigResponse(ForecastConfigBase, BaseResponseSchema, CompanyScopedMixin):
    """Schema for forecast configuration responses"""
    last_run_status: Optional[str] = Field(None, description="Last run status")
    last_run_error: Optional[str] = Field(None, description="Last run error message")
    accuracy_mape: Optional[Decimal] = Field(None, description="Forecast accuracy (MAPE)")
    created_by_id: int
    model_config = ConfigDict(from_attributes=True)


class ForecastPoint(BaseModel):
    """Single forecast data point"""
    period_date: date = Field(..., description="Period date")
    period_label: str = Field(..., description="Period label")
    forecast_value: Decimal = Field(..., description="Forecasted value")
    lower_bound_80: Optional[Decimal] = Field(None, description="Lower bound (80% CI)")
    upper_bound_80: Optional[Decimal] = Field(None, description="Upper bound (80% CI)")
    lower_bound_95: Optional[Decimal] = Field(None, description="Lower bound (95% CI)")
    upper_bound_95: Optional[Decimal] = Field(None, description="Upper bound (95% CI)")
    actual_value: Optional[Decimal] = Field(None, description="Actual value if available")
    is_historical: bool = Field(default=False, description="Whether this is historical data")


class ForecastRunBase(BaseModel):
    """Base schema for forecast runs"""
    config_id: int = Field(..., description="Forecast configuration ID")
    run_date: datetime = Field(..., description="Run timestamp")
    run_type: str = Field(default="scheduled", description="Run type: scheduled, manual, triggered")
    status: str = Field(default="running", description="Status: running, completed, failed")
    
    # Model info
    method_used: str = Field(..., description="Method used for this run")
    model_version: Optional[str] = Field(None, description="Model version used")
    
    # Training metrics
    training_start_date: Optional[date] = Field(None, description="Training data start")
    training_end_date: Optional[date] = Field(None, description="Training data end")
    training_periods: int = Field(default=0, description="Number of training periods")
    
    # Accuracy metrics
    mape: Optional[Decimal] = Field(None, description="Mean Absolute Percentage Error")
    mae: Optional[Decimal] = Field(None, description="Mean Absolute Error")
    rmse: Optional[Decimal] = Field(None, description="Root Mean Squared Error")
    
    # Forecast period
    forecast_start_date: date = Field(..., description="Forecast start date")
    forecast_end_date: date = Field(..., description="Forecast end date")
    forecast_periods: int = Field(default=0, description="Number of forecast periods")
    
    # Summary
    total_forecast: Decimal = Field(default=Decimal('0'), description="Total forecast value")
    avg_forecast: Optional[Decimal] = Field(None, description="Average forecast value")
    trend_direction: Optional[str] = Field(None, description="Trend: increasing, decreasing, stable")
    seasonality_detected: bool = Field(default=False, description="Whether seasonality was detected")
    
    # Error handling
    error_message: Optional[str] = Field(None, description="Error message if failed")
    completed_at: Optional[datetime] = Field(None, description="Completion timestamp")
    execution_time_seconds: Optional[int] = Field(None, description="Execution time")


class ForecastRunCreate(ForecastRunBase):
    """Schema for creating forecast runs"""
    pass


class ForecastRunUpdate(BaseModel):
    """Schema for updating forecast runs"""
    status: Optional[str] = None
    error_message: Optional[str] = Field(None, max_length=5000)


class ForecastRunResponse(ForecastRunBase, BaseResponseSchema, CompanyScopedMixin):
    """Schema for forecast run responses"""
    forecast_points: List[ForecastPoint] = Field(default_factory=list, description="Forecast data points")
    model_components: Optional[Dict[str, Any]] = Field(None, description="Model components (trend, seasonality, etc.)")
    diagnostics: Optional[Dict[str, Any]] = Field(None, description="Diagnostic information")
    initiated_by_id: int
    model_config = ConfigDict(from_attributes=True)


# ==================== AI INSIGHTS ====================

class AIInsightBase(BaseModel):
    """Base schema for AI insights"""
    title: str = Field(..., min_length=1, max_length=300, description="Insight title")
    description: str = Field(..., min_length=1, max_length=5000, description="Insight description")
    insight_type: InsightType = Field(..., description="Type of insight")
    
    # Source
    source_module: str = Field(..., description="Source module: accounting, treasury, hr, etc.")
    source_entity_type: Optional[str] = Field(None, description="Source entity type")
    source_entity_id: Optional[int] = Field(None, description="Source entity ID")
    
    # Impact
    impact_level: str = Field(default="medium", description="Impact level: low, medium, high")
    impact_area: Optional[str] = Field(None, description="Impact area: revenue, cost, risk, efficiency")
    estimated_impact_value: Optional[Decimal] = Field(None, description="Estimated financial impact")
    
    # Confidence and validity
    confidence_score: Decimal = Field(default=Decimal('0.8'), ge=0, le=1, description="Confidence score")
    valid_from: date = Field(default_factory=date.today, description="Valid from date")
    valid_until: Optional[date] = Field(None, description="Valid until date")
    
    # Related entities
    related_entities: Optional[List[Dict[str, Any]]] = Field(None, description="Related entities")
    supporting_data: Optional[Dict[str, Any]] = Field(None, description="Supporting data")
    
    # Actions
    suggested_actions: Optional[List[str]] = Field(None, description="Suggested actions")
    related_recommendation_ids: Optional[List[int]] = Field(None, description="Related recommendations")
    
    # Status
    is_read: bool = Field(default=False, description="Whether insight has been read")
    is_actioned: bool = Field(default=False, description="Whether action was taken")
    dismissed_at: Optional[datetime] = Field(None, description="When dismissed")
    dismissed_by_id: Optional[int] = Field(None, description="Who dismissed")


class AIInsightCreate(AIInsightBase):
    """Schema for creating AI insights"""
    pass


class AIInsightUpdate(BaseModel):
    """Schema for updating AI insights"""
    is_read: Optional[bool] = None
    is_actioned: Optional[bool] = None
    dismissed_at: Optional[datetime] = None


class AIInsightResponse(AIInsightBase, BaseResponseSchema, CompanyScopedMixin):
    """Schema for AI insight responses"""
    generated_by_model: Optional[str] = None
    generation_method: str = Field(default="automated", description="automated, manual, hybrid")
    model_config = ConfigDict(from_attributes=True)


# ==================== RECOMMENDATIONS ====================

class RecommendationBase(BaseModel):
    """Base schema for AI recommendations"""
    title: str = Field(..., min_length=1, max_length=300, description="Recommendation title")
    description: str = Field(..., min_length=1, max_length=5000, description="Recommendation description")
    recommendation_type: RecommendationType = Field(..., description="Type of recommendation")
    priority: RecommendationPriority = Field(default=RecommendationPriority.MEDIUM, description="Priority level")
    
    # Impact analysis
    estimated_benefit: Optional[Decimal] = Field(None, description="Estimated benefit value")
    estimated_cost: Optional[Decimal] = Field(None, description="Estimated implementation cost")
    estimated_roi: Optional[Decimal] = Field(None, description="Estimated ROI percentage")
    impact_timeframe: Optional[str] = Field(None, description="Impact timeframe: immediate, short_term, long_term")
    
    # Confidence
    confidence_score: Decimal = Field(default=Decimal('0.8'), ge=0, le=1, description="Confidence score")
    risk_level: Optional[str] = Field(None, description="Implementation risk: low, medium, high")
    
    # Source and context
    source_module: str = Field(..., description="Source module")
    source_insight_ids: Optional[List[int]] = Field(None, description="Source insight IDs")
    affected_entities: Optional[List[Dict[str, Any]]] = Field(None, description="Affected entities")
    
    # Implementation
    implementation_steps: Optional[List[str]] = Field(None, description="Implementation steps")
    prerequisites: Optional[List[str]] = Field(None, description="Prerequisites")
    estimated_effort: Optional[str] = Field(None, description="Estimated effort: hours, days, weeks")
    assigned_to_id: Optional[int] = Field(None, description="Assigned user ID")
    
    # Status
    status: str = Field(default="new", description="Status: new, acknowledged, in_progress, implemented, dismissed, expired")
    acknowledged_at: Optional[datetime] = Field(None, description="Acknowledgement timestamp")
    acknowledged_by_id: Optional[int] = Field(None, description="Who acknowledged")
    
    # Implementation tracking
    implemented_at: Optional[datetime] = Field(None, description="Implementation timestamp")
    implemented_by_id: Optional[int] = Field(None, description="Who implemented")
    actual_benefit: Optional[Decimal] = Field(None, description="Actual realized benefit")
    implementation_notes: Optional[str] = Field(None, max_length=5000, description="Implementation notes")
    
    # Dismissal
    dismissed_at: Optional[datetime] = Field(None, description="Dismissal timestamp")
    dismissed_by_id: Optional[int] = Field(None, description="Who dismissed")
    dismissal_reason: Optional[str] = Field(None, max_length=1000, description="Dismissal reason")
    
    # Validity
    valid_until: Optional[datetime] = Field(None, description="Expiration timestamp")


class RecommendationCreate(RecommendationBase):
    """Schema for creating recommendations"""
    pass


class RecommendationUpdate(BaseModel):
    """Schema for updating recommendations"""
    priority: Optional[RecommendationPriority] = None
    status: Optional[str] = None
    assigned_to_id: Optional[int] = None
    implementation_notes: Optional[str] = Field(None, max_length=5000)
    actual_benefit: Optional[Decimal] = None
    dismissal_reason: Optional[str] = Field(None, max_length=1000)


class RecommendationResponse(RecommendationBase, BaseResponseSchema, CompanyScopedMixin):
    """Schema for recommendation responses"""
    generated_by_model: Optional[str] = None
    assigned_to_name: Optional[str] = None
    acknowledged_by_name: Optional[str] = None
    implemented_by_name: Optional[str] = None
    dismissed_by_name: Optional[str] = None
    days_since_generated: int = Field(default=0, description="Days since generation")
    model_config = ConfigDict(from_attributes=True)


# ==================== AI ALERTS ====================

class AIAlertBase(BaseModel):
    """Base schema for AI alerts"""
    alert_type: AlertType = Field(..., description="Type of alert")
    severity: AlertSeverity = Field(default=AlertSeverity.WARNING, description="Alert severity")
    title: str = Field(..., min_length=1, max_length=300, description="Alert title")
    message: str = Field(..., min_length=1, max_length=5000, description="Alert message")
    
    # Source
    source_module: str = Field(..., description="Source module")
    source_entity_type: Optional[str] = Field(None, description="Source entity type")
    source_entity_id: Optional[int] = Field(None, description="Source entity ID")
    related_anomaly_id: Optional[int] = Field(None, description="Related anomaly ID")
    related_forecast_id: Optional[int] = Field(None, description="Related forecast ID")
    related_insight_id: Optional[int] = Field(None, description="Related insight ID")
    
    # Threshold details
    threshold_value: Optional[Decimal] = Field(None, description="Threshold value")
    actual_value: Optional[Decimal] = Field(None, description="Actual value")
    threshold_breach_percentage: Optional[Decimal] = Field(None, description="Threshold breach %")
    
    # Context
    context_data: Optional[Dict[str, Any]] = Field(None, description="Additional context data")
    
    # Notification
    notified_users: Optional[List[int]] = Field(None, description="User IDs notified")
    notification_channels: Optional[List[str]] = Field(None, description="Channels used: email, sms, in_app")
    notification_sent_at: Optional[datetime] = Field(None, description="Notification timestamp")
    
    # Status
    is_read: bool = Field(default=False, description="Whether alert has been read")
    read_at: Optional[datetime] = Field(None, description="Read timestamp")
    read_by_id: Optional[int] = Field(None, description="User who read")
    is_acknowledged: bool = Field(default=False, description="Whether acknowledged")
    acknowledged_at: Optional[datetime] = Field(None, description="Acknowledgement timestamp")
    acknowledged_by_id: Optional[int] = Field(None, description="User who acknowledged")
    is_resolved: bool = Field(default=False, description="Whether resolved")
    resolved_at: Optional[datetime] = Field(None, description="Resolution timestamp")
    resolution_notes: Optional[str] = Field(None, max_length=2000, description="Resolution notes")


class AIAlertCreate(AIAlertBase):
    """Schema for creating AI alerts"""
    pass


class AIAlertUpdate(BaseModel):
    """Schema for updating AI alerts"""
    is_read: Optional[bool] = None
    is_acknowledged: Optional[bool] = None
    is_resolved: Optional[bool] = None
    resolution_notes: Optional[str] = Field(None, max_length=2000)


class AIAlertResponse(AIAlertBase, BaseResponseSchema, CompanyScopedMixin):
    """Schema for AI alert responses"""
    generated_at: datetime = Field(default_factory=datetime.utcnow, description="Generation timestamp")
    model_config = ConfigDict(from_attributes=True)


# ==================== DOCUMENT ANALYSIS ====================

class DocumentAnalysisRequestBase(BaseModel):
    """Base schema for document analysis requests"""
    analysis_type: AnalysisType = Field(..., description="Type of analysis")
    document_id: Optional[int] = Field(None, description="Document ID if stored in system")
    document_url: Optional[str] = Field(None, max_length=500, description="Document URL")
    document_content: Optional[str] = Field(None, description="Document text content")
    
    # Options
    extract_key_terms: bool = Field(default=True, description="Extract key terms")
    extract_dates: bool = Field(default=True, description="Extract dates")
    extract_amounts: bool = Field(default=True, description="Extract monetary amounts")
    extract_parties: bool = Field(default=True, description="Extract parties")
    extract_obligations: bool = Field(default=True, description="Extract obligations")
    risk_assessment: bool = Field(default=True, description="Perform risk assessment")
    compliance_check: bool = Field(default=False, description="Perform compliance check")
    
    # Custom extraction
    custom_extractions: Optional[List[str]] = Field(None, description="Custom fields to extract")
    
    # Comparison
    compare_with_template_id: Optional[int] = Field(None, description="Template to compare against")
    compare_with_document_id: Optional[int] = Field(None, description="Document to compare against")


class DocumentAnalysisRequestCreate(DocumentAnalysisRequestBase):
    """Schema for creating document analysis requests"""
    pass


class ExtractedTerm(BaseModel):
    """Extracted term from document"""
    term_name: str = Field(..., description="Term name")
    term_value: Any = Field(..., description="Term value")
    confidence: Decimal = Field(..., ge=0, le=1, description="Extraction confidence")
    location: Optional[Dict[str, int]] = Field(None, description="Location in document (page, position)")
    context: Optional[str] = Field(None, description="Surrounding context")


class DocumentRisk(BaseModel):
    """Document risk assessment"""
    risk_type: str = Field(..., description="Type of risk")
    risk_level: str = Field(..., description="Risk level: low, medium, high, critical")
    description: str = Field(..., description="Risk description")
    affected_section: Optional[str] = Field(None, description="Affected section")
    mitigation_suggestion: Optional[str] = Field(None, description="Suggested mitigation")
    confidence: Decimal = Field(..., ge=0, le=1, description="Risk assessment confidence")


class ComplianceIssue(BaseModel):
    """Compliance issue found in document"""
    issue_type: str = Field(..., description="Type of compliance issue")
    regulation_reference: Optional[str] = Field(None, description="Regulation reference")
    description: str = Field(..., description="Issue description")
    severity: str = Field(..., description="Severity: low, medium, high")
    affected_section: Optional[str] = Field(None, description="Affected section")
    remediation_suggestion: Optional[str] = Field(None, description="Suggested remediation")


class DocumentAnalysisResult(BaseModel):
    """Document analysis result"""
    analysis_type: AnalysisType
    summary: str = Field(..., description="Document summary")
    key_terms: List[ExtractedTerm] = Field(default_factory=list, description="Extracted key terms")
    dates: List[ExtractedTerm] = Field(default_factory=list, description="Extracted dates")
    amounts: List[ExtractedTerm] = Field(default_factory=list, description="Extracted amounts")
    parties: List[ExtractedTerm] = Field(default_factory=list, description="Extracted parties")
    obligations: List[ExtractedTerm] = Field(default_factory=list, description="Extracted obligations")
    risks: List[DocumentRisk] = Field(default_factory=list, description="Risk assessments")
    compliance_issues: List[ComplianceIssue] = Field(default_factory=list, description="Compliance issues")
    overall_risk_score: Optional[Decimal] = Field(None, ge=0, le=100, description="Overall risk score")
    sentiment_score: Optional[Decimal] = Field(None, ge=-1, le=1, description="Document sentiment score")
    custom_extractions: Optional[Dict[str, Any]] = Field(None, description="Custom extraction results")
    comparison_result: Optional[Dict[str, Any]] = Field(None, description="Comparison result")
    processing_time_seconds: Optional[int] = Field(None, description="Processing time")


class DocumentAnalysisRequestResponse(DocumentAnalysisRequestBase, BaseResponseSchema, CompanyScopedMixin):
    """Schema for document analysis request responses"""
    status: str = Field(default="pending", description="Status: pending, processing, completed, failed")
    result: Optional[DocumentAnalysisResult] = Field(None, description="Analysis result")
    error_message: Optional[str] = Field(None, description="Error message if failed")
    requested_by_id: int
    completed_at: Optional[datetime] = Field(None, description="Completion timestamp")
    model_config = ConfigDict(from_attributes=True)


# ==================== NATURAL LANGUAGE COMMANDS ====================

class NLCommandRequestBase(BaseModel):
    """Base schema for natural language command requests"""
    command_text: str = Field(..., min_length=1, max_length=5000, description="Natural language command")
    context: Optional[Dict[str, Any]] = Field(None, description="Additional context")
    conversation_history: Optional[List[Dict[str, str]]] = Field(None, description="Previous conversation")
    
    # Execution options
    auto_execute: bool = Field(default=False, description="Automatically execute if safe")
    require_confirmation: bool = Field(default=True, description="Require user confirmation")


class NLCommandRequestCreate(NLCommandRequestBase):
    """Schema for creating NL command requests"""
    pass


class NLCommandAction(BaseModel):
    """Parsed action from natural language"""
    action_type: str = Field(..., description="Action type: create, update, delete, query, analyze, etc.")
    target_entity: str = Field(..., description="Target entity")
    parameters: Dict[str, Any] = Field(default_factory=dict, description="Action parameters")
    filters: Optional[Dict[str, Any]] = Field(None, description="Query filters")
    confidence: Decimal = Field(..., ge=0, le=1, description="Parsing confidence")


class NLCommandRequestResponse(NLCommandRequestBase, BaseResponseSchema):
    """Schema for NL command request responses"""
    status: str = Field(default="pending", description="Status: pending, parsed, executed, failed")
    
    # Interpretation
    intent: Optional[str] = Field(None, description="Detected intent")
    entities: Optional[Dict[str, Any]] = Field(None, description="Extracted entities")
    actions: Optional[List[NLCommandAction]] = Field(None, description="Parsed actions")
    interpretation_confidence: Optional[Decimal] = Field(None, description="Interpretation confidence")
    
    # Clarification needed
    needs_clarification: bool = Field(default=False, description="Whether clarification needed")
    clarification_question: Optional[str] = Field(None, description="Question for user")
    suggested_interpretations: Optional[List[str]] = Field(None, description="Suggested interpretations")
    
    # Execution
    is_safe_to_execute: bool = Field(default=False, description="Whether safe to execute")
    risk_level: Optional[str] = Field(None, description="Risk level: safe, low, medium, high")
    requires_approval: bool = Field(default=False, description="Whether requires approval")
    
    # Result
    execution_result: Optional[Dict[str, Any]] = Field(None, description="Execution result")
    response_text: Optional[str] = Field(None, description="Human-readable response")
    error_message: Optional[str] = Field(None, description="Error message if failed")
    
    requested_by_id: int
    executed_at: Optional[datetime] = Field(None, description="Execution timestamp")
    model_config = ConfigDict(from_attributes=True)


# ==================== EXPLAINABILITY ====================

class AIExplanationBase(BaseModel):
    """Base schema for AI explanations"""
    entity_type: str = Field(..., description="Entity type being explained")
    entity_id: int = Field(..., description="Entity ID")
    explanation_type: str = Field(..., description="Explanation type: prediction, recommendation, anomaly, decision")
    
    # Summary
    summary: str = Field(..., description="Human-readable explanation summary")
    technical_explanation: Optional[str] = Field(None, description="Technical explanation")
    
    # Feature contributions
    feature_contributions: Optional[List[Dict[str, Any]]] = Field(None, description="Feature contribution to decision")
    
    # SHAP values or similar
    shap_values: Optional[Dict[str, Decimal]] = Field(None, description="SHAP values")
    
    # Decision path
    decision_path: Optional[List[Dict[str, Any]]] = Field(None, description="Decision path taken")
    
    # Confidence factors
    confidence_factors: Optional[Dict[str, Decimal]] = Field(None, description="Factors affecting confidence")
    
    # Similar cases
    similar_cases: Optional[List[Dict[str, Any]]] = Field(None, description="Similar historical cases")
    
    # Model info
    model_used: Optional[str] = Field(None, description="Model used for decision")
    model_version: Optional[str] = Field(None, description="Model version")


class AIExplanationCreate(AIExplanationBase):
    """Schema for creating AI explanations"""
    pass


class AIExplanationResponse(AIExplanationBase, BaseResponseSchema):
    """Schema for AI explanation responses"""
    generated_at: datetime = Field(default_factory=datetime.utcnow)
    model_config = ConfigDict(from_attributes=True)


# ==================== FILTER PARAMETERS ====================

class AIModelConfigFilterParams(PaginationParams):
    """Filter parameters for AI model config queries"""
    model_type: Optional[AIModelType] = None
    status: Optional[ModelStatus] = None
    is_active: Optional[bool] = None


class DetectedAnomalyFilterParams(PaginationParams):
    """Filter parameters for detected anomaly queries"""
    anomaly_type: Optional[AnomalyType] = None
    severity: Optional[AnomalySeverity] = None
    status: Optional[str] = None
    entity_type: Optional[str] = None
    detected_from: Optional[datetime] = None
    detected_to: Optional[datetime] = None


class ForecastConfigFilterParams(PaginationParams):
    """Filter parameters for forecast config queries"""
    forecast_type: Optional[ForecastType] = None
    is_active: Optional[bool] = None


class ForecastRunFilterParams(PaginationParams):
    """Filter parameters for forecast run queries"""
    config_id: Optional[int] = None
    status: Optional[str] = None
    run_from: Optional[datetime] = None
    run_to: Optional[datetime] = None


class AIInsightFilterParams(PaginationParams):
    """Filter parameters for AI insight queries"""
    insight_type: Optional[InsightType] = None
    source_module: Optional[str] = None
    is_read: Optional[bool] = None
    is_actioned: Optional[bool] = None


class RecommendationFilterParams(PaginationParams):
    """Filter parameters for recommendation queries"""
    recommendation_type: Optional[RecommendationType] = None
    priority: Optional[RecommendationPriority] = None
    status: Optional[str] = None
    assigned_to_id: Optional[int] = None


class AIAlertFilterParams(PaginationParams):
    """Filter parameters for AI alert queries"""
    alert_type: Optional[AlertType] = None
    severity: Optional[AlertSeverity] = None
    is_read: Optional[bool] = None
    is_resolved: Optional[bool] = None
    source_module: Optional[str] = None


class DocumentAnalysisFilterParams(PaginationParams):
    """Filter parameters for document analysis queries"""
    analysis_type: Optional[AnalysisType] = None
    status: Optional[str] = None


# ==================== LIST RESPONSES ====================

class AIModelConfigListResponse(ListResponseBase):
    """List response for AI model configs"""
    items: List[AIModelConfigResponse]


class AnomalyDetectionConfigListResponse(ListResponseBase):
    """List response for anomaly detection configs"""
    items: List[AnomalyDetectionConfigResponse]


class DetectedAnomalyListResponse(ListResponseBase):
    """List response for detected anomalies"""
    items: List[DetectedAnomalyResponse]


class ForecastConfigListResponse(ListResponseBase):
    """List response for forecast configs"""
    items: List[ForecastConfigResponse]


class ForecastRunListResponse(ListResponseBase):
    """List response for forecast runs"""
    items: List[ForecastRunResponse]


class AIInsightListResponse(ListResponseBase):
    """List response for AI insights"""
    items: List[AIInsightResponse]


class RecommendationListResponse(ListResponseBase):
    """List response for recommendations"""
    items: List[RecommendationResponse]


class AIAlertListResponse(ListResponseBase):
    """List response for AI alerts"""
    items: List[AIAlertResponse]


class DocumentAnalysisListResponse(ListResponseBase):
    """List response for document analysis requests"""
    items: List[DocumentAnalysisRequestResponse]


class NLCommandListResponse(ListResponseBase):
    """List response for NL command requests"""
    items: List[NLCommandRequestResponse]


class AIExplanationListResponse(ListResponseBase):
    """List response for AI explanations"""
    items: List[AIExplanationResponse]


# ==================== ANALYTICS RESPONSES ====================

class AIStatisticsResponse(BaseModel):
    """AI module statistics"""
    total_models: int = 0
    active_models: int = 0
    total_predictions: int = 0
    predictions_today: int = 0
    avg_prediction_latency_ms: int = 0
    
    # Anomalies
    total_anomalies_detected: int = 0
    anomalies_last_30_days: int = 0
    open_anomalies: int = 0
    critical_anomalies: int = 0
    
    # Forecasts
    active_forecasts: int = 0
    forecasts_run_today: int = 0
    avg_forecast_accuracy: Optional[Decimal] = None
    
    # Insights
    total_insights: int = 0
    unread_insights: int = 0
    insights_last_7_days: int = 0
    
    # Recommendations
    total_recommendations: int = 0
    new_recommendations: int = 0
    implemented_recommendations: int = 0
    total_realized_benefit: Decimal = Decimal('0')
    
    # Alerts
    active_alerts: int = 0
    unacknowledged_alerts: int = 0
    alerts_last_24_hours: int = 0
    
    # Document analysis
    documents_analyzed: int = 0
    avg_analysis_time_seconds: int = 0
    
    # NL commands
    nl_commands_today: int = 0
    nl_commands_success_rate: Optional[Decimal] = None


class AIModelPerformanceHistory(BaseModel):
    """AI model performance over time"""
    model_id: int
    model_name: str
    measurement_date: date
    accuracy: Optional[Decimal] = None
    precision: Optional[Decimal] = None
    recall: Optional[Decimal] = None
    f1_score: Optional[Decimal] = None
    prediction_count: int = 0
    error_count: int = 0
    avg_latency_ms: int = 0


class AIModelComparisonResponse(BaseModel):
    """AI model comparison"""
    models: List[AIModelConfigResponse]
    comparison_metrics: List[str]
    comparison_data: List[Dict[str, Any]]
    best_model_by_metric: Dict[str, int]