content_parser.py

"""HTML content parsing and cleaning utilities."""
from bs4 import BeautifulSoup
from playwright.sync_api import Page
from rich.console import Console
import re

console = Console()

class ProfileParser:
    """Parses candidate profile HTML and extracts structured data."""
    
    MAX_PROFILE_LENGTH = 2500  # Character limit for profile text
    
    def __init__(self):
        """Initialize parser."""
        pass
    
    def parse_profile(self, page: Page) -> dict:
        """
        Extract profile information from a candidate page.
        Returns dict with: name, bio, skills, experience, location, interests.
        """
        try:
            # Get page HTML
            html_content = page.content()
            soup = BeautifulSoup(html_content, 'html.parser')
            
            # Extract profile data (selectors will need adjustment based on actual HTML)
            profile = {
                'name': self._extract_name(soup, page),
                'bio': self._extract_bio(soup, page),
                'skills': self._extract_skills(soup, page),
                'experience': self._extract_experience(soup, page),
                'location': self._extract_location(soup, page),
                'interests': self._extract_interests(soup, page),
            }
            
            # Build combined profile text for LLM
            profile['full_text'] = self._build_profile_text(profile)
            
            return profile
            
        except Exception as e:
            console.print(f"[red]Error parsing profile: {e}[/red]")
            return self._empty_profile()
    
    def _extract_name(self, soup: BeautifulSoup, page: Page) -> str:
        """Extract candidate name."""
        try:
            # Try common selectors
            name_elem = (
                soup.find('h1') or 
                soup.find(class_=re.compile(r'name', re.I)) or
                soup.find(attrs={'data-testid': re.compile(r'name', re.I)})
            )
            if name_elem:
                return self._clean_text(name_elem.get_text())
            
            # Fallback: try Playwright selector
            name_selector = page.query_selector('h1')
            if name_selector:
                return self._clean_text(name_selector.inner_text())
            
            return "Unknown"
        except:
            return "Unknown"
    
    def _extract_bio(self, soup: BeautifulSoup, page: Page) -> str:
        """Extract candidate bio/about section."""
        try:
            # Try common bio selectors
            bio_elem = (
                soup.find(class_=re.compile(r'bio|about|description', re.I)) or
                soup.find('p', class_=re.compile(r'profile|intro', re.I))
            )
            if bio_elem:
                return self._clean_text(bio_elem.get_text())
            
            # Fallback: get all paragraphs and combine
            paragraphs = soup.find_all('p')
            if paragraphs:
                bio_text = ' '.join([p.get_text() for p in paragraphs[:3]])
                return self._clean_text(bio_text)
            
            return ""
        except:
            return ""
    
    def _extract_skills(self, soup: BeautifulSoup, page: Page) -> list:
        """Extract list of skills."""
        try:
            skills = []
            
            # Try finding skill tags/badges
            skill_elems = soup.find_all(class_=re.compile(r'skill|tag|badge', re.I))
            for elem in skill_elems:
                skill = self._clean_text(elem.get_text())
                if skill and len(skill) < 50:  # Reasonable skill length
                    skills.append(skill)
            
            return skills[:10]  # Limit to 10 skills
        except:
            return []
    
    def _extract_experience(self, soup: BeautifulSoup, page: Page) -> str:
        """Extract work experience."""
        try:
            exp_elem = soup.find(class_=re.compile(r'experience|work|background', re.I))
            if exp_elem:
                return self._clean_text(exp_elem.get_text())
            return ""
        except:
            return ""
    
    def _extract_location(self, soup: BeautifulSoup, page: Page) -> str:
        """Extract location."""
        try:
            loc_elem = soup.find(class_=re.compile(r'location|city|place', re.I))
            if loc_elem:
                return self._clean_text(loc_elem.get_text())
            return ""
        except:
            return ""
    
    def _extract_interests(self, soup: BeautifulSoup, page: Page) -> str:
        """Extract interests/focus areas."""
        try:
            int_elem = soup.find(class_=re.compile(r'interest|focus|passion', re.I))
            if int_elem:
                return self._clean_text(int_elem.get_text())
            return ""
        except:
            return ""
    
    def _clean_text(self, text: str) -> str:
        """Clean and normalize text content."""
        if not text:
            return ""
        
        # Remove extra whitespace
        text = re.sub(r'\s+', ' ', text)
        text = text.strip()
        
        return text
    
    def _build_profile_text(self, profile: dict) -> str:
        """Build combined profile text for LLM evaluation."""
        parts = []
        
        if profile['name']:
            parts.append(f"Name: {profile['name']}")
        
        if profile['bio']:
            parts.append(f"Bio: {profile['bio']}")
        
        if profile['skills']:
            parts.append(f"Skills: {', '.join(profile['skills'])}")
        
        if profile['experience']:
            parts.append(f"Experience: {profile['experience']}")
        
        if profile['location']:
            parts.append(f"Location: {profile['location']}")
        
        if profile['interests']:
            parts.append(f"Interests: {profile['interests']}")
        
        full_text = '\n'.join(parts)
        
        # Truncate if too long
        if len(full_text) > self.MAX_PROFILE_LENGTH:
            full_text = full_text[:self.MAX_PROFILE_LENGTH]
            console.print(f"[yellow]Profile truncated to {self.MAX_PROFILE_LENGTH} chars[/yellow]")
        
        return full_text
    
    def _empty_profile(self) -> dict:
        """Return empty profile structure."""
        return {
            'name': 'Unknown',
            'bio': '',
            'skills': [],
            'experience': '',
            'location': '',
            'interests': '',
            'full_text': ''
        }
    
    def estimate_tokens(self, text: str) -> int:
        """Estimate token count for text."""
        return int(len(text.split()) * 1.3)