multi_portfolio_manager.py

"""
Multi-Portfolio Risk Management Service

Provides advanced risk management across multiple portfolios including:
- Portfolio aggregation and correlation analysis
- Cross-portfolio risk assessment
- Multi-portfolio optimization
- Risk concentration detection
"""

import numpy as np
import pandas as pd
from typing import List, Dict, Optional, Tuple
from datetime import datetime, timedelta
import logging
from scipy import optimize
from dataclasses import dataclass

logger = logging.getLogger(__name__)

@dataclass
class Portfolio:
    id: int
    name: str
    holdings: List[Dict]
    user_id: int
    created_at: datetime = None
    updated_at: datetime = None

class MultiPortfolioRiskManager:
    """Multi-portfolio risk management and optimization service"""
    
    def __init__(self, risk_free_rate: float = 0.02):
        self.risk_free_rate = risk_free_rate
        self.logger = logging.getLogger(__name__)
    
    def aggregate_portfolios(self, portfolios: List[Portfolio]) -> Dict:
        """Aggregate multiple portfolios into consolidated view"""
        
        try:
            consolidated_holdings = {}
            total_value = 0
            portfolio_correlations = {}
            
            # Aggregate holdings across all portfolios
            for portfolio in portfolios:
                portfolio_value = sum(h.get('current_value', 0) for h in portfolio.holdings)
                total_value += portfolio_value
                
                for holding in portfolio.holdings:
                    ticker = holding.get('ticker')
                    current_value = holding.get('current_value', 0)
                    
                    if ticker not in consolidated_holdings:
                        consolidated_holdings[ticker] = {
                            'ticker': ticker,
                            'total_shares': 0,
                            'current_value': 0,
                            'portfolios': [],
                            'weight': 0
                        }
                    
                    consolidated_holdings[ticker]['total_shares'] += holding.get('shares', 0)
                    consolidated_holdings[ticker]['current_value'] += current_value
                    consolidated_holdings[ticker]['portfolios'].append({
                        'portfolio_id': portfolio.id,
                        'portfolio_name': portfolio.name,
                        'shares': holding.get('shares', 0),
                        'value': current_value
                    })
            
            # Calculate weights
            for ticker, holding in consolidated_holdings.items():
                holding['weight'] = holding['current_value'] / total_value if total_value > 0 else 0
            
            # Calculate portfolio correlations
            portfolio_correlations = self._calculate_portfolio_correlations(portfolios)
            
            # Analyze concentration risk
            concentration_analysis = self._analyze_concentration_risk(consolidated_holdings)
            
            # Analyze cross-portfolio exposures
            cross_exposures = self._analyze_cross_exposures(consolidated_holdings, portfolios)
            
            # Calculate diversification benefits
            diversification_benefits = self._calculate_diversification_benefits(portfolios, consolidated_holdings)
            
            return {
                'total_portfolios': len(portfolios),
                'total_value': total_value,
                'consolidated_holdings': consolidated_holdings,
                'portfolio_correlations': portfolio_correlations,
                'concentration_analysis': concentration_analysis,
                'cross_exposures': cross_exposures,
                'diversification_benefits': diversification_benefits,
                'aggregation_timestamp': datetime.now().isoformat()
            }
            
        except Exception as e:
            self.logger.error(f"Portfolio aggregation failed: {e}")
            raise
    
    def calculate_portfolio_var(self, portfolios: List[Portfolio], confidence_level: float = 0.95, time_horizon: int = 1) -> Dict:
        """Calculate Value at Risk across multiple portfolios"""
        
        try:
            # Aggregate portfolios
            aggregation = self.aggregate_portfolios(portfolios)
            consolidated_holdings = aggregation['consolidated_holdings']
            
            if not consolidated_holdings:
                return {'error': 'No holdings found'}
            
            # Get tickers for historical data
            tickers = list(consolidated_holdings.keys())
            
            # Get historical returns data
            returns_data = self._get_historical_returns(tickers, period='1y')
            
            if returns_data.empty:
                return {'error': 'Could not fetch historical data'}
            
            # Calculate portfolio weights
            weights = np.array([consolidated_holdings[ticker]['weight'] for ticker in tickers if ticker in returns_data.columns])
            available_tickers = [ticker for ticker in tickers if ticker in returns_data.columns]
            
            if len(available_tickers) == 0:
                return {'error': 'No valid tickers for analysis'}
            
            # Normalize weights
            weights = weights / np.sum(weights) if np.sum(weights) > 0 else weights
            
            # Calculate portfolio returns
            portfolio_returns = (returns_data[available_tickers] * weights).sum(axis=1)
            
            # Calculate VaR using multiple methods
            var_results = self._calculate_var_multiple_methods(portfolio_returns, confidence_level, time_horizon)
            
            # Calculate individual portfolio VaRs
            individual_vars = {}
            for portfolio in portfolios:
                individual_var = self._calculate_individual_portfolio_var(portfolio, returns_data, confidence_level, time_horizon)
                individual_vars[portfolio.name] = individual_var
            
            # Calculate diversification benefit
            total_individual_var = sum(individual_vars.values())
            diversification_benefit = total_individual_var - abs(var_results['parametric_var']) if total_individual_var > 0 else 0
            
            return {
                'consolidated_var': var_results,
                'individual_portfolio_vars': individual_vars,
                'diversification_benefit': diversification_benefit,
                'total_portfolio_value': aggregation['total_value'],
                'var_as_percentage': (abs(var_results['parametric_var']) / aggregation['total_value'] * 100) if aggregation['total_value'] > 0 else 0,
                'analysis_date': datetime.now().isoformat()
            }
            
        except Exception as e:
            self.logger.error(f"VaR calculation failed: {e}")
            raise
    
    def optimize_multi_portfolio_allocation(self, portfolios: List[Portfolio], 
                                          target_return: float = None,
                                          max_risk: float = None,
                                          constraints: Dict = None) -> Dict:
        """Optimize allocation across multiple portfolios"""
        
        try:
            # Aggregate portfolios
            aggregation = self.aggregate_portfolios(portfolios)
            consolidated_holdings = aggregation['consolidated_holdings']
            
            if not consolidated_holdings:
                return {'optimization_successful': False, 'error': 'No holdings found'}
            
            tickers = list(consolidated_holdings.keys())
            
            # Get historical returns
            returns_data = self._get_historical_returns(tickers, period='1y')
            
            if returns_data.empty:
                return {'optimization_successful': False, 'error': 'Could not fetch historical data'}
            
            # Calculate expected returns and covariance matrix
            expected_returns = returns_data.mean() * 252  # Annualize
            cov_matrix = returns_data.cov() * 252  # Annualize
            
            # Current weights
            current_weights = np.array([consolidated_holdings.get(ticker, {}).get('weight', 0) for ticker in returns_data.columns])
            
            # Normalize current weights
            current_weights = current_weights / np.sum(current_weights) if np.sum(current_weights) > 0 else current_weights
            
            # Set up optimization constraints
            constraints_list = [{'type': 'eq', 'fun': lambda x: np.sum(x) - 1}]  # Weights sum to 1
            
            # Bounds (non-negative weights)
            bounds = tuple((0, 1) for _ in range(len(returns_data.columns)))
            
            if constraints:
                if 'max_weight' in constraints:
                    max_weight = constraints['max_weight']
                    bounds = tuple((0, max_weight) for _ in range(len(returns_data.columns)))
            
            # Objective functions
            if target_return:
                # Minimize risk for target return
                def objective(weights):
                    return np.sqrt(np.dot(weights.T, np.dot(cov_matrix, weights)))
                
                constraints_list.append({
                    'type': 'eq', 
                    'fun': lambda x: np.dot(x, expected_returns) - target_return
                })
            elif max_risk:
                # Maximize return with risk constraint
                def objective(weights):
                    return -np.dot(weights, expected_returns)
                
                constraints_list.append({
                    'type': 'ineq',
                    'fun': lambda x: max_risk - np.sqrt(np.dot(x.T, np.dot(cov_matrix, x)))
                })
            else:
                # Maximize Sharpe ratio
                def objective(weights):
                    portfolio_return = np.dot(weights, expected_returns)
                    portfolio_vol = np.sqrt(np.dot(weights.T, np.dot(cov_matrix, weights)))
                    return -(portfolio_return - self.risk_free_rate) / portfolio_vol
            
            # Run optimization
            result = optimize.minimize(
                objective, 
                current_weights, 
                method='SLSQP',
                bounds=bounds,
                constraints=constraints_list
            )
            
            if result.success:
                optimal_weights = result.x
                
                # Calculate metrics for optimal portfolio
                optimal_return = np.dot(optimal_weights, expected_returns)
                optimal_vol = np.sqrt(np.dot(optimal_weights.T, np.dot(cov_matrix, optimal_weights)))
                optimal_sharpe = (optimal_return - self.risk_free_rate) / optimal_vol
                
                # Compare with current allocation
                current_return = np.dot(current_weights, expected_returns)
                current_vol = np.sqrt(np.dot(current_weights.T, np.dot(cov_matrix, current_weights)))
                current_sharpe = (current_return - self.risk_free_rate) / current_vol
                
                return {
                    'optimization_successful': True,
                    'optimal_weights': dict(zip(returns_data.columns, optimal_weights)),
                    'current_weights': dict(zip(returns_data.columns, current_weights)),
                    'optimal_metrics': {
                        'expected_return': optimal_return,
                        'volatility': optimal_vol,
                        'sharpe_ratio': optimal_sharpe
                    },
                    'current_metrics': {
                        'expected_return': current_return,
                        'volatility': current_vol,
                        'sharpe_ratio': current_sharpe
                    },
                    'improvement': {
                        'return_improvement': optimal_return - current_return,
                        'vol_improvement': current_vol - optimal_vol,
                        'sharpe_improvement': optimal_sharpe - current_sharpe
                    }
                }
            else:
                return {
                    'optimization_successful': False,
                    'error': result.message
                }
                
        except Exception as e:
            self.logger.error(f"Portfolio optimization failed: {e}")
            raise
    
    # Helper methods
    
    def _calculate_portfolio_correlations(self, portfolios: List[Portfolio]) -> Dict:
        """Calculate correlations between portfolios"""
        if len(portfolios) < 2:
            return {}
        
        # Simplified correlation calculation
        correlations = {}
        for i, port1 in enumerate(portfolios):
            for j, port2 in enumerate(portfolios[i+1:], i+1):
                # Calculate overlap-based correlation (simplified)
                overlap = self._calculate_portfolio_overlap(port1, port2)
                correlations[f"{port1.name}_{port2.name}"] = overlap
        
        return correlations
    
    def _calculate_portfolio_overlap(self, port1: Portfolio, port2: Portfolio) -> float:
        """Calculate overlap between two portfolios"""
        tickers1 = set(h['ticker'] for h in port1.holdings)
        tickers2 = set(h['ticker'] for h in port2.holdings)
        
        intersection = len(tickers1 & tickers2)
        union = len(tickers1 | tickers2)
        
        return intersection / union if union > 0 else 0
    
    def _analyze_concentration_risk(self, consolidated_holdings: Dict) -> Dict:
        """Analyze position concentration risk"""
        weights = [holding['weight'] for holding in consolidated_holdings.values()]
        
        # Herfindahl-Hirschman Index
        hhi = sum(w**2 for w in weights)
        
        # Number of effective positions
        effective_positions = 1 / hhi if hhi > 0 else 0
        
        return {
            'hhi': hhi,
            'effective_positions': effective_positions,
            'concentration_level': (
                'High' if hhi > 0.25 else 
                'Medium' if hhi > 0.1 else 
                'Low'
            )
        }
    
    def _analyze_cross_exposures(self, consolidated_holdings: Dict, portfolios: List[Portfolio]) -> Dict:
        """Analyze cross-portfolio exposures"""
        cross_exposures = {}
        
        for ticker, holding in consolidated_holdings.items():
            if len(holding['portfolios']) > 1:
                cross_exposures[ticker] = {
                    'total_weight': holding['weight'],
                    'portfolio_count': len(holding['portfolios']),
                    'portfolio_breakdown': holding['portfolios']
                }
        
        return cross_exposures
    
    def _calculate_diversification_benefits(self, portfolios: List[Portfolio], consolidated_holdings: Dict) -> Dict:
        """Calculate diversification benefits from portfolio aggregation"""
        # Simplified diversification benefit calculation
        total_positions = len(consolidated_holdings)
        total_portfolios = len(portfolios)
        
        # Estimate diversification score
        diversification_score = min(total_positions / (total_portfolios * 10), 1.0)
        
        return {
            'diversification_score': diversification_score,
            'total_unique_positions': total_positions,
            'average_positions_per_portfolio': total_positions / total_portfolios if total_portfolios > 0 else 0
        }
    
    def _get_historical_returns(self, tickers: List[str], period: str = '1y') -> pd.DataFrame:
        """Get historical returns for list of tickers"""
        import yfinance as yf
        
        try:
            data = yf.download(tickers, period=period, progress=False)
            if len(tickers) == 1:
                prices = data['Close']
            else:
                prices = data['Close']
            
            returns = prices.pct_change().dropna()
            return returns
        except:
            # Return empty DataFrame if fetch fails
            return pd.DataFrame()
    
    def _calculate_var_multiple_methods(self, returns: pd.Series, confidence_level: float, time_horizon: int) -> Dict:
        """Calculate VaR using multiple methods"""
        # Historical VaR
        hist_var = np.percentile(returns, (1 - confidence_level) * 100) * np.sqrt(time_horizon)
        
        # Parametric VaR (assuming normal distribution)
        mean_return = returns.mean()
        std_return = returns.std()
        from scipy import stats
        z_score = stats.norm.ppf(1 - confidence_level)
        param_var = (mean_return + z_score * std_return) * np.sqrt(time_horizon)
        
        return {
            'historical_var': hist_var,
            'parametric_var': param_var,
            'confidence_level': confidence_level,
            'time_horizon': time_horizon
        }
    
    def _calculate_individual_portfolio_var(self, portfolio: Portfolio, returns_data: pd.DataFrame, confidence_level: float, time_horizon: int) -> float:
        """Calculate VaR for individual portfolio"""
        # Get portfolio weights
        portfolio_tickers = [h['ticker'] for h in portfolio.holdings]
        portfolio_weights = []
        
        total_value = sum(h.get('current_value', 0) for h in portfolio.holdings)
        
        for holding in portfolio.holdings:
            weight = holding.get('current_value', 0) / total_value if total_value > 0 else 0
            portfolio_weights.append(weight)
        
        # Filter returns data
        available_tickers = [t for t in portfolio_tickers if t in returns_data.columns]
        if not available_tickers:
            return 0
        
        available_weights = np.array([
            portfolio_weights[i] for i, t in enumerate(portfolio_tickers) if t in available_tickers
        ])
        
        # Normalize weights
        if np.sum(available_weights) > 0:
            available_weights = available_weights / np.sum(available_weights)
        
        # Calculate portfolio returns
        portfolio_returns = (returns_data[available_tickers] * available_weights).sum(axis=1)
        
        # Calculate VaR
        var_result = self._calculate_var_multiple_methods(portfolio_returns, confidence_level, time_horizon)
        return var_result['parametric_var']