strategy_engine.py

"""
Strategy Engine Module

Enhanced trading strategy engine with backtesting and performance tracking.
"""

import yfinance as yf
from ta.trend import SMAIndicator, MACD, ADXIndicator
from ta.momentum import RSIIndicator
from ta.volatility import BollingerBands, AverageTrueRange
import time
from typing import Dict, Any, Optional, List
import pandas as pd

import config
from logger import trading_logger, system_logger
from position_tracker import PositionTracker
from trade_queue import TradeQueue


class StrategyEngine:
    """
    Enhanced trading strategy engine with signal generation and backtesting.
    """
    
    def __init__(self, trader, data_manager):
        """
        Initialize strategy engine.
        
        Args:
            trader: AlpacaTrader instance
            data_manager: DataManager instance
        """
        self.trader = trader
        self.data_manager = data_manager
        
        # Strategy parameters
        self.max_positions = config.DEFAULT_MAX_POSITIONS
        self.position_size = config.DEFAULT_POSITION_SIZE
        self.stop_loss = config.DEFAULT_STOP_LOSS
        self.take_profit = config.DEFAULT_TAKE_PROFIT
        
        # Position tracking for trailing stops (NEW)
        self.position_tracker = PositionTracker()
        
        # Trade queue for after-hours orders (NEW)
        self.trade_queue = TradeQueue()
        
        # Performance tracking
        self.signals_generated = 0
        self.trades_executed = 0
        
        system_logger.info("Strategy engine initialized with trailing stops and trade queue")
    
    def update_parameters(
        self,
        max_positions: Optional[int] = None,
        position_size: Optional[float] = None,
        stop_loss: Optional[float] = None,
        take_profit: Optional[float] = None
    ):
        """
        Update strategy parameters.
        
        Args:
            max_positions: Maximum number of positions
            position_size: Position size as percentage of portfolio
            stop_loss: Stop loss percentage
            take_profit: Take profit percentage
        """
        if max_positions is not None:
            self.max_positions = max_positions
        if position_size is not None:
            self.position_size = min(position_size, config.MAX_POSITION_SIZE)
        if stop_loss is not None:
            self.stop_loss = stop_loss
        if take_profit is not None:
            self.take_profit = take_profit
        
        trading_logger.info(
            f"Strategy parameters updated: max_pos={self.max_positions}, "
            f"size={self.position_size*100:.1f}%, sl={self.stop_loss*100:.1f}%, "
            f"tp={self.take_profit*100:.1f}%"
        )
    
    def calculate_indicators(self, df: pd.DataFrame) -> Optional[pd.DataFrame]:
        """
        Calculate technical indicators for a dataframe.
        
        Args:
            df: DataFrame with OHLCV data
            
        Returns:
            DataFrame with indicators added or None if error
        """
        if df is None or len(df) < 50:
            return None
        
        try:
            # Simple Moving Averages
            df['SMA_20'] = SMAIndicator(df['Close'], window=20).sma_indicator()
            df['SMA_50'] = SMAIndicator(df['Close'], window=50).sma_indicator()
            
            # MACD
            macd = MACD(df['Close'])
            df['MACD'] = macd.macd()
            df['MACD_Signal'] = macd.macd_signal()
            df['MACD_Hist'] = macd.macd_diff()
            
            # RSI
            df['RSI'] = RSIIndicator(df['Close'], window=14).rsi()
            
            # Bollinger Bands
            bollinger = BollingerBands(df['Close'])
            df['BB_High'] = bollinger.bollinger_hband()
            df['BB_Low'] = bollinger.bollinger_lband()
            df['BB_Mid'] = bollinger.bollinger_mavg()
            
            # Volume
            df['Volume_SMA'] = df['Volume'].rolling(window=20).mean()
            
            # Price momentum
            df['Returns'] = df['Close'].pct_change()
            df['Momentum'] = df['Close'].pct_change(periods=10)
            
            # ATR - Average True Range (for volatility measurement)
            atr = AverageTrueRange(df['High'], df['Low'], df['Close'], window=14)
            df['ATR'] = atr.average_true_range()
            df['ATR_Percent'] = (df['ATR'] / df['Close']) * 100  # ATR as % of price
            
            # ADX - Average Directional Index (for trend strength)
            adx = ADXIndicator(df['High'], df['Low'], df['Close'], window=14)
            df['ADX'] = adx.adx()
            df['ADX_Pos'] = adx.adx_pos()  # Positive directional indicator
            df['ADX_Neg'] = adx.adx_neg()  # Negative directional indicator
            
            return df
            
        except Exception as e:
            system_logger.error(f"Error calculating indicators: {str(e)}")
            return None
    
    def generate_signal(self, symbol: str) -> Optional[Dict[str, Any]]:
        """
        Generate trading signal for a symbol using technical analysis.
        
        Args:
            symbol: Stock symbol
            
        Returns:
            Signal dictionary or None if error
        """
        try:
            df = self.data_manager.get_historical_data(symbol, 180)
            
            if df is None or len(df) < 50:
                return None
            
            df = self.calculate_indicators(df)
            if df is None:
                return None
            
            latest = df.iloc[-1]
            prev = df.iloc[-2]
            
            # Check for data quality
            if pd.isna(latest['ADX']) or pd.isna(latest['ATR']):
                system_logger.warning(f"Missing ADX or ATR data for {symbol}")
                return None
            
            score = 0
            signals = []
            confidence_factors = []
            
            # ENHANCED ENTRY FILTERS (NEW - improves win rate)
            
            # Volume Confirmation Filter
            volume_confirmed = latest['Volume'] >= latest['Volume_SMA'] * config.MIN_VOLUME_MULTIPLE
            if not volume_confirmed and config.REQUIRE_VOLUME_CONFIRMATION:
                # Insufficient volume - likely false breakout
                return None
            
            # ADX Rising Filter (trend is strengthening, not weakening)
            if len(df) >= 3:
                prev_adx = df.iloc[-2]['ADX']
                prev_prev_adx = df.iloc[-3]['ADX']
                adx_rising = latest['ADX'] > prev_adx and prev_adx >= prev_prev_adx
            else:
                adx_rising = True  # Not enough data, allow trade
            
            if not adx_rising and config.REQUIRE_ADX_RISING:
                # ADX falling - trend weakening, avoid entry
                signals.append(f"ADX falling ({latest['ADX']:.1f}→{prev_adx:.1f}) - skip")
                return None
            
            # ADX Trend Strength Filter (CRITICAL - filters weak trends)
            adx_value = latest['ADX']
            trend_strength = "WEAK"
            
            if adx_value < 20:
                # Weak/choppy market - reduce all signals
                trend_strength = "WEAK"
                score_multiplier = 0.5  # Reduce score in choppy markets
            elif adx_value < 25:
                # Developing trend
                trend_strength = "DEVELOPING"
                score_multiplier = 0.75
            elif adx_value < 40:
                # Strong trend
                trend_strength = "STRONG"
                score_multiplier = 1.0
            else:
                # Very strong trend (possibly exhaustion)
                trend_strength = "VERY_STRONG"
                score_multiplier = 0.9
            
            # Trend Direction Analysis (Weight: 35%)
            if latest['Close'] > latest['SMA_20'] > latest['SMA_50']:
                raw_score = 4
                if latest['ADX_Pos'] > latest['ADX_Neg']:  # Confirm with ADX directional
                    raw_score += 1
                score += int(raw_score * score_multiplier)
                signals.append(f"Uptrend (SMA) - ADX:{adx_value:.1f}")
                confidence_factors.append(0.35)
            elif latest['Close'] < latest['SMA_20'] < latest['SMA_50']:
                raw_score = -4
                if latest['ADX_Neg'] > latest['ADX_Pos']:  # Confirm with ADX directional
                    raw_score -= 1
                score += int(raw_score * score_multiplier)
                signals.append(f"Downtrend (SMA) - ADX:{adx_value:.1f}")
                confidence_factors.append(0.35)
            
            # MACD Analysis (Weight: 25%)
            if latest['MACD'] > latest['MACD_Signal'] and prev['MACD'] <= prev['MACD_Signal']:
                score += int(4 * score_multiplier)
                signals.append("MACD bullish crossover")
                confidence_factors.append(0.25)
            elif latest['MACD'] < latest['MACD_Signal'] and prev['MACD'] >= prev['MACD_Signal']:
                score += int(-4 * score_multiplier)
                signals.append("MACD bearish crossover")
                confidence_factors.append(0.25)
            elif latest['MACD'] > latest['MACD_Signal']:
                score += int(2 * score_multiplier)
                signals.append("MACD bullish")
                confidence_factors.append(0.15)
            else:
                score += int(-2 * score_multiplier)
                confidence_factors.append(0.15)
            
            # RSI Analysis (Weight: 20%)
            if latest['RSI'] < 30:
                score += int(3 * score_multiplier)
                signals.append(f"RSI oversold ({latest['RSI']:.1f})")
                confidence_factors.append(0.2)
            elif latest['RSI'] > 70:
                score += int(-3 * score_multiplier)
                signals.append(f"RSI overbought ({latest['RSI']:.1f})")
                confidence_factors.append(0.2)
            elif 40 < latest['RSI'] < 60:
                # Neutral RSI
                confidence_factors.append(0.1)
            
            # Bollinger Bands (Weight: 15%)
            if latest['Close'] < latest['BB_Low']:
                score += int(2 * score_multiplier)
                signals.append("Below lower BB")
                confidence_factors.append(0.15)
            elif latest['Close'] > latest['BB_High']:
                score += int(-2 * score_multiplier)
                signals.append("Above upper BB")
                confidence_factors.append(0.15)
            
            # Volume Analysis (Weight: 10%)
            if latest['Volume'] > latest['Volume_SMA'] * 1.5:
                if score > 0:
                    score += int(2 * score_multiplier)
                    signals.append("High volume confirmation")
                else:
                    score += int(-2 * score_multiplier)
                    signals.append("High volume warning")
                confidence_factors.append(0.1)
            
            # Determine action with stricter thresholds
            # Only trade in strong enough trends (ADX > 20)
            if adx_value >= 20:
                if score >= 8:
                    action = 'BUY'
                elif score <= -8:
                    action = 'SELL'
                else:
                    action = 'HOLD'
            else:
                # Choppy market - don't trade
                action = 'HOLD'
                signals.append(f"Market too choppy (ADX: {adx_value:.1f})")
            
            # Calculate confidence (0-100)
            max_score = 15  # New maximum possible score
            base_confidence = min(abs(score) / max_score * 100, 100)
            
            # Boost confidence for strong trends
            if adx_value > 30:
                base_confidence *= 1.2
            
            confidence = min(base_confidence, 100)
            
            signal = {
                'action': action,
                'score': score,
                'price': latest['Close'],
                'confidence': confidence,
                'signals': signals,
                'rsi': latest['RSI'],
                'macd': latest['MACD'],
                'macd_signal': latest['MACD_Signal'],
                'sma_20': latest['SMA_20'],
                'sma_50': latest['SMA_50'],
                'volume_ratio': latest['Volume'] / latest['Volume_SMA'] if latest['Volume_SMA'] > 0 else 1,
                'adx': adx_value,
                'trend_strength': trend_strength,
                'atr': latest['ATR'],
                'atr_percent': latest['ATR_Percent']
            }
            
            self.signals_generated += 1
            
            system_logger.debug(
                f"Signal generated for {symbol}: {action} (score={score}, conf={confidence:.1f}%)"
            )
            
            return signal
            
        except Exception as e:
            system_logger.error(f"Error generating signal for {symbol}: {str(e)}")
            return None
    
    def monitor_positions(self):
        """
        Monitor open positions and update trailing stops.
        
        Called periodically to update trailing stops and execute exits.
        """
        try:
            if not hasattr(self, 'position_tracker'):
                return
            
            tracked_symbols = self.position_tracker.get_all_positions()
            if not tracked_symbols:
                return  # No positions to monitor
            
            for symbol in tracked_symbols:
                try:
                    # Get current price
                    quote = self.trader.get_quote(symbol)
                    if not quote:
                        continue
                    
                    current_price = float(quote['price'])
                    
                    # Update position and check for stop hit
                    update_info = self.position_tracker.update_position(symbol, current_price)
                    
                    if update_info:
                        # Check for partial profit target hit (NEW)
                        if update_info.get('partial_target_hit'):
                            partial_qty = update_info.get('partial_qty')
                            exit_price = update_info.get('exit_price')
                            partial_pnl = update_info.get('partial_pnl', 0)
                            
                            trading_logger.info(
                                f"💰 Partial profit target for {symbol} - selling {partial_qty} shares @ ${exit_price:.2f}"
                            )
                            
                            try:
                                # Execute partial exit
                                order = self.trader.place_order(
                                    symbol=symbol,
                                    quantity=partial_qty,
                                    side='sell',
                                    order_type='market'
                                )
                                
                                trading_logger.info(
                                    f"✅ Partial exit {symbol}: Sold {partial_qty} shares @ ${exit_price:.2f} "
                                    f"(Profit: ${partial_pnl:.2f}). Remaining position trails with stop."
                                )
                                
                                # Update position quantity in tracker
                                pos_info = self.position_tracker.get_position_info(symbol)
                                if pos_info:
                                    remaining_qty = pos_info['quantity'] - partial_qty
                                    self.position_tracker.update_quantity(symbol, remaining_qty)
                                
                            except Exception as e:
                                trading_logger.error(f"Failed to execute partial exit for {symbol}: {e}")
                        
                        # Check if stop was hit
                        elif update_info.get('stop_hit'):
                            trading_logger.warning(
                                f"🛑 Trailing stop HIT for {symbol} at ${current_price:.2f}"
                            )
                            
                            # Get position details
                            pos_info = self.position_tracker.get_position_info(symbol)
                            if pos_info:
                                quantity = pos_info['quantity']
                                
                                # Execute exit
                                try:
                                    order = self.trader.place_order(
                                        symbol=symbol,
                                        quantity=quantity,
                                        side='sell',
                                        order_type='market'
                                    )
                                    
                                    pnl = update_info.get('pnl', 0)
                                    trading_logger.info(
                                        f"✅ Exited {symbol}: {quantity} shares @ ${current_price:.2f} "
                                        f"(P&L: ${pnl:.2f})"
                                    )
                                    
                                    # Remove from tracker
                                    self.position_tracker.remove_position(symbol)
                                    
                                except Exception as e:
                                    trading_logger.error(f"Failed to execute stop exit for {symbol}: {e}")
                        
                        # Stop was raised (locked in more profit)
                        elif update_info.get('new_stop'):
                            locked_profit = update_info.get('locked_profit', 0)
                            system_logger.debug(
                                f"📈 {symbol}: Stop raised to ${update_info['new_stop']:.2f} "
                                f"(Locked: ${locked_profit:.2f})"
                            )
                            
                except Exception as e:
                    system_logger.error(f"Error monitoring {symbol}: {e}")
                    continue
            
        except Exception as e:
            system_logger.error(f"Error in monitor_positions: {e}")
    
    def execute_queued_trades(self):
        """
        Execute all queued trades (to be called at market open).
        
        Returns:
            Number of trades executed
        """
        try:
            if not hasattr(self, 'trade_queue'):
                return 0
            
            executed_count = self.trade_queue.execute_all_pending(self.trader)
            
            if executed_count > 0:
                trading_logger.info(
                    f"🚀 Executed {executed_count} queued trades at market open"
                )
                
                # Clear executed trades from queue
                self.trade_queue.clear_executed()
            
            return executed_count
            
        except Exception as e:
            system_logger.error(f"Error executing queued trades: {e}")
            return 0
    
    def scan_opportunities(self) -> List[Dict[str, Any]]:
        """
        Scan stock universe for trading opportunities.
        
        Returns:
            List of opportunities sorted by confidence
        """
        opportunities = []
        
        trading_logger.info(f"Scanning {len(self.data_manager.stock_universe)} symbols for opportunities...")
        
        for symbol in self.data_manager.stock_universe.keys():
            try:
                signal = self.generate_signal(symbol)
                
                if signal and signal['action'] in ['BUY', 'SELL']:
                    opportunities.append({
                        'symbol': symbol,
                        'name': self.data_manager.stock_universe[symbol],
                        **signal
                    })
                
                # Rate limiting
                time.sleep(0.5)
                
            except Exception as e:
                system_logger.warning(f"Error scanning {symbol}: {str(e)}")
                continue
        
        # Sort by confidence
        opportunities.sort(key=lambda x: x['confidence'], reverse=True)
        
        trading_logger.info(f"Found {len(opportunities)} trading opportunities")
        
        return opportunities
    
    def execute_strategy(self, dry_run: bool = False) -> Dict[str, Any]:
        """
        Execute trading strategy based on signals.
        
        Args:
            dry_run: If True, only simulate trades
            
        Returns:
            Dictionary with execution results
        """
        try:
            # Get current positions
            positions = self.trader.get_positions()
            current_symbols = {p['symbol'] for p in positions}
            
            # Get account info
            account = self.trader.get_account_info()
            buying_power = account['buying_power']
            equity = account['equity']
            
            # Check if we can add more positions
            if len(positions) >= self.max_positions:
                trading_logger.info(
                    f"Max positions ({self.max_positions}) reached, skipping new trades"
                )
                return {
                    'status': 'max_positions_reached',
                    'current_positions': len(positions),
                    'max_positions': self.max_positions
                }
            
            # Scan for opportunities
            opportunities = self.scan_opportunities()
            
            executed_trades = []
            
            # Execute trades for top opportunities
            for opp in opportunities:
                if len(current_symbols) >= self.max_positions:
                    break
                
                symbol = opp['symbol']
                action = opp['action']
                
                # Only buy if we don't have this position
                if action == 'BUY' and symbol not in current_symbols:
                    price = opp['price']
                    atr = opp.get('atr', price * 0.02)  # Fallback to 2% if no ATR
                    atr_percent = opp.get('atr_percent', 2.0)
                    
                    # ATR-BASED POSITION SIZING
                    # Risk a fixed percentage of equity per trade (default 1%)
                    risk_per_trade = equity * 0.01  # 1% risk per trade
                    
                    # Calculate position size based on ATR
                    # Stop will be placed at 2x ATR below entry
                    atr_stop_distance = atr * 2.0
                    
                    # Position size = Risk Amount / Stop Distance
                    # This ensures consistent risk regardless of volatility
                    qty_by_risk = int(risk_per_trade / atr_stop_distance)
                    
                    # Also limit by maximum position size (% of equity)
                    max_position_value = equity * self.position_size
                    qty_by_max_size = int(max_position_value / price)
                    
                    # Use the smaller of the two to ensure both constraints
                    qty = min(qty_by_risk, qty_by_max_size)
                    
                    if qty > 0:
                        # Calculate ATR-based dynamic targets
                        # Stop loss: 2x ATR below entry (adapts to volatility)
                        stop_loss = price - (atr * 2.0)
                        
                        # Take profit: 3x ATR above entry (2:1 reward:risk at minimum)
                        # Or use configured take_profit if larger
                        atr_take_profit = price + (atr * 3.0)
                        pct_take_profit = price * (1 + self.take_profit)
                        take_profit = max(atr_take_profit, pct_take_profit)
                        
                        if dry_run:
                            trading_logger.info(
                                f"DRY RUN: Would BUY {qty} {symbol} @ ${price:.2f} "
                                f"(ATR: ${atr:.2f}, {atr_percent:.1f}% | "
                                f"TP: ${take_profit:.2f}, SL: ${stop_loss:.2f})"
                            )
                        else:
                            # Check if market is open (NEW - queue if closed)
                            if not self.trade_queue.is_market_open():
                                # Queue trade for market open
                                self.trade_queue.add_trade(
                                    symbol=symbol,
                                    quantity=qty,
                                    side='buy',
                                    price=price,
                                    stop_loss=stop_loss,
                                    take_profit=take_profit,
                                    reason="Market closed"
                                )
                                trading_logger.info(
                                    f"⏰ Trade queued (market closed): BUY {qty} {symbol} @ ${price:.2f}"
                                )
                                continue
                            
                            try:
                                # Place bracket order with ATR-based stops
                                order = self.trader.place_bracket_order(
                                    symbol=symbol,
                                    quantity=qty,
                                    side='buy',
                                    take_profit_price=take_profit,
                                    stop_loss_price=stop_loss
                                )
                                
                                executed_trades.append({
                                    'symbol': symbol,
                                    'action': 'BUY',
                                    'quantity': qty,
                                    'price': price,
                                    'atr': atr,
                                    'atr_percent': atr_percent,
                                    'stop_loss': stop_loss,
                                    'take_profit': take_profit,
                                    'risk_reward_ratio': (take_profit - price) / (price - stop_loss),
                                    'order_id': order['id']
                                })
                                
                                # Track position for trailing stops (NEW)
                                self.position_tracker.add_position(
                                    symbol=symbol,
                                    entry_price=price,
                                    quantity=qty,
                                    atr=atr,
                                    initial_stop=stop_loss
                                )
                                
                                current_symbols.add(symbol)
                                self.trades_executed += 1
                                
                                trading_logger.info(
                                    f"✓ EXECUTED: BUY {qty} {symbol} @ ${price:.2f} "
                                    f"(ADX: {opp.get('adx', 0):.1f}, ATR: {atr_percent:.1f}%) - Trailing stops active"
                                )
                                
                            except Exception as e:
                                system_logger.error(f"Failed to execute trade for {symbol}: {str(e)}")
            
            return {
                'status': 'completed',
                'opportunities_found': len(opportunities),
                'trades_executed': len(executed_trades),
                'trades': executed_trades,
                'dry_run': dry_run
            }
            
        except Exception as e:
            system_logger.error(f"Strategy execution error: {str(e)}")
            return {
                'status': 'error',
                'error': str(e)
            }
    
    def backtest_signal(
        self,
        symbol: str,
        start_date: str,
        end_date: str
    ) -> Optional[Dict[str, Any]]:
        """
        Backtest strategy on historical data.
        
        Args:
            symbol: Stock symbol
            start_date: Start date (YYYY-MM-DD)
            end_date: End date (YYYY-MM-DD)
            
        Returns:
            Backtest results dictionary
        """
        try:
            # This is a simplified backtest - production would be more sophisticated
            ticker = yf.Ticker(symbol)
            df = ticker.history(start=start_date, end=end_date)
            
            if df is None or len(df) < 50:
                return None
            
            df = self.calculate_indicators(df)
            if df is None:
                return None
            
            # Simple backtest logic
            trades = 0
            wins = 0
            total_return = 0
            
            # This is a placeholder for actual backtesting logic
            # Production implementation would track entries/exits properly
            
            return {
                'symbol': symbol,
                'period': f"{start_date} to {end_date}",
                'total_trades': trades,
                'winning_trades': wins,
                'total_return': total_return
            }
            
        except Exception as e:
            system_logger.error(f"Backtest error for {symbol}: {str(e)}")
            return None
    
    def get_performance_stats(self) -> Dict[str, Any]:
        """
        Get strategy performance statistics.
        
        Returns:
            Performance statistics dictionary
        """
        return {
            'signals_generated': self.signals_generated,
            'trades_executed': self.trades_executed,
            'max_positions': self.max_positions,
            'position_size': self.position_size,
            'stop_loss': self.stop_loss,
            'take_profit': self.take_profit
        }