Boundary Detection System Design¶

# Example: Transaction boundary detection
def find_transaction_boundaries(document_text):
    """
    Identify boundaries by finding:
    1. Last transaction of current statement
    2. First header line of next statement
    """

    # Statement end patterns
    end_patterns = [
        r'(?i)closing\s+balance',
        r'(?i)balance\s+brought\s+forward',
        r'(?i)total\s+(?:debits|credits)',
        r'(?i)end\s+of\s+statement'
    ]

    # Statement start patterns
    start_patterns = [
        r'(?i)statement\s+(?:period|from|for)',
        r'(?i)account\s+(?:number|holder)',
        r'(?i)page\s+1\s+of\s+\d+',
        r'(?i)(?:opening|previous)\s+balance'
    ]

class BoundaryDetectionProvider:
    """Abstract interface for boundary detection providers"""

    def analyze_boundaries(self, text: str, **kwargs) -> BoundaryResult:
        """Detect statement boundaries in document text"""
        pass

    def is_available(self) -> bool:
        """Check if provider is available and configured"""
        pass

class OpenAIBoundaryDetection:
    def analyze_boundaries(self, text: str, **kwargs) -> BoundaryResult:
        system_msg = SystemMessage(content="""
        You are a financial document analyzer specializing in bank statements.
        Analyze the provided text and identify individual bank statement boundaries.

        Each statement typically:
        1. Starts with bank header/logo information
        2. Contains account holder details
        3. Has a statement period or date range
        4. Includes transaction listings
        5. Ends with balance summaries

        Look for these boundary indicators:
        - Bank headers and letterheads
        - Different account numbers
        - New statement periods (From/To dates)
        - Page numbering resets (Page 1 of X)
        - Changes in customer names or addresses
        """)

        human_msg = HumanMessage(content=f"""
        Analyze this bank statement document ({total_pages} pages) and identify
        all individual statement boundaries.

        DOCUMENT TEXT:
        {text_sample}

        Provide precise page boundaries for each detected statement.
        """)

        response = self.llm.invoke([system_msg, human_msg])
        return self._parse_structured_response(response.content)

class OllamaBoundaryDetection:
    def analyze_boundaries(self, text: str, **kwargs) -> BoundaryResult:
        prompt = f"""
        You are a bank statement analyzer. Analyze this document and identify
        individual bank statement boundaries.

        DOCUMENT TEXT ({total_pages} pages):
        {text}

        Look for these boundary indicators:
        - Bank headers and letterheads (e.g., "NAB", "Westpac", "Commonwealth Bank")
        - Different account numbers (even if similar)
        - New statement periods (different date ranges)
        - Page numbering resets (Page 1 of X starting over)
        - Account type changes (e.g., "iSaver" vs "Visa Credit")

        RESPONSE FORMAT: Return ONLY a valid JSON object:
        {{
            "total_statements": <number>,
            "boundaries": [
                {{
                    "start_page": <number>,
                    "end_page": <number>,
                    "account_number": "<identifier_if_found>"
                }}
            ]
        }}
        """

        message = HumanMessage(content=prompt)
        response = self.llm.invoke([message])
        return self._parse_json_response(response.content)

def detect_statement_boundaries(self, text_chunks, total_pages):
    """Multi-provider boundary detection with fallback strategy"""

    # Try primary LLM provider
    if self.provider and self.provider.is_available():
        try:
            text = self._prepare_text_for_analysis(text_chunks)
            result = self.provider.analyze_boundaries(text, total_pages=total_pages)

            # Validate results through hallucination detection
            if self._validate_llm_results(result, total_pages, text):
                return self._convert_provider_boundaries(result, total_pages)

        except LLMProviderError as e:
            logger.warning(f"Primary LLM provider failed: {e}")

    # Fallback to content-based detection
    logger.info("Falling back to content-based boundary detection")
    return self._detect_content_based_boundaries(text_chunks, total_pages)

def _detect_page_markers(self, combined_text: str, total_pages: int) -> List[StatementBoundary]:
    """Detect boundaries using 'Page 1 of X' markers"""

    # Find all page markers
    page_pattern = r'(?i)page\s+(\d+)\s+of\s+(\d+)'
    page_matches = list(re.finditer(page_pattern, combined_text))

    # Identify "Page 1" markers (statement starts)
    page_1_positions = []
    for match in page_matches:
        page_num = int(match.group(1))
        if page_num == 1:  # Start of new statement
            char_pos = match.start()
            estimated_doc_page = max(1, int((char_pos / len(combined_text)) * total_pages))
            page_1_positions.append({
                'doc_page': estimated_doc_page,
                'statement_pages': int(match.group(2)),
                'confidence': 0.9
            })

    # Create boundaries based on page 1 markers
    boundaries = []
    for i, pos in enumerate(page_1_positions):
        start_page = pos['doc_page']
        if i < len(page_1_positions) - 1:
            end_page = max(start_page, page_1_positions[i + 1]['doc_page'] - 1)
        else:
            end_page = total_pages

        boundaries.append(StatementBoundary(
            start_page, end_page,
            confidence=pos['confidence'],
            reasoning=f"Page marker: 'Page 1 of {pos['statement_pages']}' found"
        ))

    return boundaries

def _detect_account_boundaries(self, combined_text: str, total_pages: int) -> List[StatementBoundary]:
    """Detect boundaries based on account number changes"""

    # Account number patterns
    account_patterns = [
        r'(?i)(?:account|card)\s*(?:number|no\.?)?\s*[:]\s*(\d[\d\s]{8,})',
        r'(?i)account\s*[:]\s*(\d+(?:\s+\d+)*)',
        r'(?i)card\s*number\s*[:]\s*(\d+(?:\s+\d+)*)'
    ]

    unique_accounts = {}
    for pattern in account_patterns:
        for match in re.finditer(pattern, combined_text):
            account = re.sub(r'\s', '', match.group(1))  # Remove spaces
            if len(account) >= 8:  # Valid account length
                char_pos = match.start()
                estimated_page = max(1, int((char_pos / len(combined_text)) * total_pages))
                if account not in unique_accounts:
                    unique_accounts[account] = estimated_page

    # Create boundaries based on account changes
    if len(unique_accounts) > 1:
        account_pages = sorted(unique_accounts.values())
        boundaries = []

        for i, page in enumerate(account_pages):
            start_page = page
            end_page = account_pages[i + 1] - 1 if i < len(account_pages) - 1 else total_pages

            boundaries.append(StatementBoundary(
                start_page, end_page,
                confidence=0.7,
                reasoning=f"Account change: New account detected on page {start_page}"
            ))

        return boundaries

    return []

def _detect_statement_headers(self, text_chunks: List[str], total_pages: int) -> List[StatementBoundary]:
    """Detect boundaries using statement header patterns"""

    header_patterns = [
        # Statement period indicators
        r'(?i)statement\s+(?:period|from|for)',
        r'(?i)statement\s+(?:date|issued)',

        # Account information headers
        r'(?i)account\s+(?:holder|name)',
        r'(?i)(?:bsb|routing|sort)\s+code',

        # Balance indicators (often at statement start)
        r'(?i)(?:opening|previous|starting)\s+balance',

        # Institution-specific headers
        r'(?i)businesschoice|complete\s+access|classic\s+banking',
        r'(?i)everyday\s+account|isaver|goal\s+saver'
    ]

    statement_starts = []
    combined_text = " ".join(text_chunks)
    lines = combined_text.split('\n')

    for i, line in enumerate(lines):
        # Score line based on header patterns
        header_score = sum(1 for pattern in header_patterns
                          if re.search(pattern, line))

        # Require multiple pattern matches for confidence
        if header_score >= 2:
            estimated_page = max(1, min(total_pages, int((i / len(lines)) * total_pages) + 1))
            statement_starts.append({
                'line_index': i,
                'page': estimated_page,
                'confidence': header_score / len(header_patterns),
                'text_sample': line[:100]
            })

    # Convert header positions to boundaries
    if len(statement_starts) > 1:
        boundaries = []
        for i, start in enumerate(statement_starts):
            start_page = start['page']
            end_page = (statement_starts[i + 1]['page'] - 1
                       if i < len(statement_starts) - 1
                       else total_pages)

            if start_page <= end_page:
                boundaries.append(StatementBoundary(
                    start_page, end_page,
                    confidence=start['confidence'],
                    reasoning=f"Header detected: {start['text_sample'][:50]}..."
                ))

        return boundaries

    return []

def _apply_westpac_12_page_pattern(self, total_pages: int) -> List[StatementBoundary]:
    """Apply known Westpac 12-page document pattern"""

    if total_pages == 12:
        logger.info("Applying Westpac 12-page pattern segmentation")

        return [
            StatementBoundary(1, 2, confidence=0.7,
                            reasoning="Pattern: Westpac billing statement (pages 1-2)"),
            StatementBoundary(3, 5, confidence=0.7,
                            reasoning="Pattern: Westpac card 1 statement (pages 3-5)"),
            StatementBoundary(6, 12, confidence=0.7,
                            reasoning="Pattern: Westpac card 2 statement (pages 6-12)")
        ]

    return []

def _apply_heuristic_segmentation(self, total_pages: int) -> List[StatementBoundary]:
    """Apply statistical heuristics for multi-statement documents"""

    if total_pages >= 8:
        # Estimate pages per statement based on total document length
        estimated_pages_per_statement = min(6, max(3, total_pages // 3))

        boundaries = []
        current_page = 1
        statement_num = 1

        while current_page <= total_pages:
            end_page = min(current_page + estimated_pages_per_statement - 1, total_pages)

            boundaries.append(StatementBoundary(
                current_page, end_page,
                confidence=0.6,
                reasoning=f"Heuristic: Estimated statement {statement_num} "
                         f"(pages {current_page}-{end_page})"
            ))

            current_page = end_page + 1
            statement_num += 1

            # Safety check
            if statement_num > 10:
                break

        return boundaries

    return []

def _apply_single_statement_default(self, total_pages: int) -> List[StatementBoundary]:
    """Default to single statement spanning all pages"""

    return [StatementBoundary(
        1, total_pages,
        confidence=0.5,
        reasoning="Default: Single statement assumption (all detection methods failed)"
    )]

# BROKEN LOGIC (Fixed in August 2025)
if boundary.start_page <= last_boundary.end_page + 1:  # ❌ WRONG
    # This treated pages (1-2, 3-4) as "overlapping" when they're separate!

def _validate_and_consolidate_boundaries(self, boundaries: List[StatementBoundary],
                                       total_pages: int) -> List[StatementBoundary]:
    """Validate and consolidate overlapping or invalid boundaries"""

    if not boundaries:
        return self._apply_single_statement_default(total_pages)

    # Sort boundaries by start page
    sorted_boundaries = sorted(boundaries, key=lambda b: b.start_page)
    consolidated = []

    for boundary in sorted_boundaries:
        if not consolidated:
            consolidated.append(boundary)
            continue

        last_boundary = consolidated[-1]

        # ✅ FIXED: Check for TRUE overlap (not just adjacent pages)
        if boundary.start_page <= last_boundary.end_page:
            # TRUE overlapping boundaries - merge only if they're the same statement
            if (boundary.account_number and last_boundary.account_number and
                boundary.account_number == last_boundary.account_number):
                # Same account - merge boundaries
                logger.info(f"Merging overlapping boundaries for same account: {last_boundary.start_page}-{last_boundary.end_page} and {boundary.start_page}-{boundary.end_page}")
                consolidated[-1] = StatementBoundary(
                    start_page=last_boundary.start_page,
                    end_page=max(last_boundary.end_page, boundary.end_page),
                    account_number=boundary.account_number or last_boundary.account_number,
                    statement_period=boundary.statement_period or last_boundary.statement_period,
                    confidence=min(last_boundary.confidence, boundary.confidence),
                    reasoning=f"Merged boundaries: {last_boundary.reasoning} + {boundary.reasoning}"
                )
            elif not boundary.account_number and not last_boundary.account_number:
                # Both missing account numbers - merge cautiously
                logger.info(f"Merging overlapping boundaries with no account info: {last_boundary.start_page}-{last_boundary.end_page} and {boundary.start_page}-{boundary.end_page}")
                consolidated[-1] = StatementBoundary(
                    start_page=last_boundary.start_page,
                    end_page=max(last_boundary.end_page, boundary.end_page),
                    account_number=boundary.account_number or last_boundary.account_number,
                    statement_period=boundary.statement_period or last_boundary.statement_period,
                    confidence=min(last_boundary.confidence, boundary.confidence) * 0.8,  # Lower confidence
                    reasoning=f"Merged boundaries (no account info): {last_boundary.reasoning} + {boundary.reasoning}"
                )
            else:
                # Different accounts or mixed info - this is likely separate statements with overlap error
                logger.warning(f"Overlapping boundaries for different statements - keeping first: {last_boundary.start_page}-{last_boundary.end_page}, skipping: {boundary.start_page}-{boundary.end_page}")
                # Don't add the overlapping boundary
        else:
            # ✅ FIXED: No overlap - adjacent or separate boundaries are fine
            consolidated.append(boundary)

    # Final validation - ensure boundaries don't exceed document
    for i, boundary in enumerate(consolidated):
        if boundary.end_page > total_pages:
            consolidated[i] = StatementBoundary(
                start_page=boundary.start_page,
                end_page=total_pages,
                confidence=boundary.confidence,
                reasoning=boundary.reasoning + " (end_page corrected)"
            )

    return consolidated

def _validate_page_ranges(self, boundaries: List[StatementBoundary],
                         total_pages: int) -> List[StatementBoundary]:
    """Validate that page ranges are logically correct"""

    validated = []

    for boundary in boundaries:
        # Ensure start_page >= 1
        start_page = max(1, boundary.start_page)

        # Ensure end_page <= total_pages
        end_page = min(total_pages, boundary.end_page)

        # Ensure start_page <= end_page
        if start_page > end_page:
            logger.warning(f"Invalid boundary range {start_page}-{end_page}, skipping")
            continue

        # Create validated boundary
        validated.append(StatementBoundary(
            start_page=start_page,
            end_page=end_page,
            confidence=boundary.confidence,
            reasoning=boundary.reasoning + " (validated)"
        ))

    return validated

class DocumentTypeHandler:
    """Handle different document types and formats"""

    def detect_document_type(self, text: str, total_pages: int) -> str:
        """Identify document type based on content analysis"""

        # Westpac detection
        if re.search(r'(?i)westpac.*businesschoice', text) and total_pages == 12:
            return "westpac_12_page"

        # Commonwealth Bank detection
        if re.search(r'(?i)commonwealth.*complete\s+access', text):
            return "cba_standard"

        # NAB detection
        if re.search(r'(?i)nab.*classic\s+banking', text):
            return "nab_standard"

        # Generic multi-statement
        if total_pages >= 8:
            return "multi_statement"

        return "single_statement"

    def apply_type_specific_detection(self, document_type: str,
                                    text_chunks: List[str],
                                    total_pages: int) -> List[StatementBoundary]:
        """Apply document-type-specific detection logic"""

        if document_type == "westpac_12_page":
            return self._apply_westpac_12_page_pattern(total_pages)
        elif document_type == "multi_statement":
            return self._apply_heuristic_segmentation(total_pages)
        else:
            return self._apply_single_statement_default(total_pages)

def _handle_malformed_document(self, text_chunks: List[str],
                              total_pages: int,
                              error: Exception) -> List[StatementBoundary]:
    """Handle documents with formatting or parsing issues"""

    logger.warning(f"Document appears malformed: {error}")

    # Attempt basic content analysis
    try:
        combined_text = " ".join(text_chunks)

        # Look for any recognizable patterns
        if len(combined_text) > 1000:  # Sufficient content
            # Try simple keyword-based segmentation
            return self._attempt_keyword_segmentation(combined_text, total_pages)

    except Exception as recovery_error:
        logger.error(f"Recovery attempt failed: {recovery_error}")

    # Ultimate fallback
    return self._apply_single_statement_default(total_pages)

def _attempt_keyword_segmentation(self, text: str, total_pages: int) -> List[StatementBoundary]:
    """Last-resort segmentation using basic keywords"""

    # Look for statement keywords
    statement_keywords = [
        'statement', 'account', 'balance', 'transaction',
        'opening balance', 'closing balance', 'page 1 of'
    ]

    keyword_positions = []
    for keyword in statement_keywords:
        for match in re.finditer(re.escape(keyword), text, re.IGNORECASE):
            char_pos = match.start()
            estimated_page = max(1, int((char_pos / len(text)) * total_pages))
            keyword_positions.append(estimated_page)

    # Use keyword density to estimate boundaries
    if len(set(keyword_positions)) > 1:
        unique_positions = sorted(set(keyword_positions))
        boundaries = []

        for i, pos in enumerate(unique_positions):
            start_page = pos
            end_page = unique_positions[i + 1] - 1 if i < len(unique_positions) - 1 else total_pages

            if start_page <= end_page:
                boundaries.append(StatementBoundary(
                    start_page, end_page,
                    confidence=0.3,
                    reasoning=f"Keyword-based: Detected activity near page {start_page}"
                ))

        return boundaries

    return self._apply_single_statement_default(total_pages)

# PROBLEMATIC APPROACH (Fixed in August 2025)
def _prepare_text_for_analysis(self, text_chunks: List[str]) -> str:
    return " ".join(text_chunks)[:8000]  # ❌ No page boundaries visible to LLM

def _prepare_text_for_analysis(self, text_chunks: List[str]) -> str:
    """Prepare text for LLM analysis with clear page markers."""
    # ✅ FIXED: Add clear page markers so LLM knows page boundaries
    marked_chunks = []
    for i, chunk in enumerate(text_chunks):
        marked_chunks.append(f"=== PAGE {i+1} ===\n{chunk}\n=== END PAGE {i+1} ===")

    combined = "\n\n".join(marked_chunks)

    # ✅ IMPROVED: Smarter truncation preserving beginning and end
    if len(combined) > 12000:
        # Include first few pages and last few pages with clear truncation marker
        first_part = ""
        last_part = ""

        # Include first 2-3 pages
        for i in range(min(3, len(marked_chunks))):
            first_part += marked_chunks[i] + "\n\n"
            if len(first_part) > 6000:
                break

        # Include last 2-3 pages
        for i in range(max(0, len(marked_chunks)-3), len(marked_chunks)):
            if i >= 3:  # Don't duplicate if we already included in first_part
                last_part += marked_chunks[i] + "\n\n"
            if len(last_part) > 4000:
                break

        combined = first_part + "\n[... MIDDLE PAGES TRUNCATED ...]\n\n" + last_part

    return combined[:15000]  # ✅ INCREASED: Higher limit with better structure

# LEGACY APPROACH - KEPT FOR REFERENCE
def _prepare_text_for_analysis_legacy(self, text_chunks: List[str]) -> str:
    """Legacy text preparation - caused boundary detection issues"""

    combined_text = " ".join(text_chunks)  # No page markers

    # Simple truncation without structure preservation
    if len(combined_text) > 12000:
        beginning = combined_text[:8000]
        end = combined_text[-4000:]
        return f"{beginning}\n\n... [TRUNCATED] ...\n\n{end}"

    return combined_text[:8000]  # Hard limit caused information loss

import hashlib
from typing import Dict, Optional

class BoundaryDetectionCache:
    """Cache boundary detection results"""

    def __init__(self, max_size: int = 100):
        self.cache: Dict[str, BoundaryDetectionResult] = {}
        self.max_size = max_size
        self.access_order = []

    def _generate_cache_key(self, text_chunks: List[str], total_pages: int) -> str:
        """Generate cache key from document content"""
        combined_text = "".join(text_chunks)
        content_hash = hashlib.md5(combined_text.encode()).hexdigest()
        return f"{content_hash}_{total_pages}"

    def get(self, text_chunks: List[str], total_pages: int) -> Optional[BoundaryDetectionResult]:
        """Retrieve cached result if available"""
        cache_key = self._generate_cache_key(text_chunks, total_pages)

        if cache_key in self.cache:
            # Update access order
            self.access_order.remove(cache_key)
            self.access_order.append(cache_key)

            logger.debug(f"Cache hit for boundary detection: {cache_key}")
            return self.cache[cache_key]

        return None

    def set(self, text_chunks: List[str], total_pages: int, result: BoundaryDetectionResult):
        """Cache boundary detection result"""
        cache_key = self._generate_cache_key(text_chunks, total_pages)

        # Implement LRU eviction
        if len(self.cache) >= self.max_size:
            oldest_key = self.access_order.pop(0)
            del self.cache[oldest_key]

        self.cache[cache_key] = result
        self.access_order.append(cache_key)

        logger.debug(f"Cached boundary detection result: {cache_key}")

@dataclass
class StatementBoundary:
    """Represents a detected statement boundary"""
    start_page: int
    end_page: int
    account_number: Optional[str] = None
    statement_period: Optional[str] = None
    confidence: float = 0.8
    reasoning: str = "Boundary detection"

@dataclass
class BoundaryDetectionResult:
    """Result from boundary detection analysis"""
    total_statements: int
    boundaries: List[StatementBoundary]
    analysis_notes: Optional[str] = None
    detection_method: str = "unknown"
    processing_time: float = 0.0

class LLMAnalyzer:
    """Main boundary detection coordinator"""

    def __init__(self, config: Any, provider: Optional[LLMProvider] = None):
        self.config = config
        self.provider = provider or LLMProviderFactory.create_from_config(config)
        self.cache = BoundaryDetectionCache()
        self.document_type_handler = DocumentTypeHandler()

    def detect_statement_boundaries(self, text_chunks: List[str],
                                  total_pages: int) -> BoundaryDetectionResult:
        """Main entry point for boundary detection"""

        start_time = time.time()

        # Check cache first
        cached_result = self.cache.get(text_chunks, total_pages)
        if cached_result:
            return cached_result

        # Detect document type
        combined_text = " ".join(text_chunks)
        document_type = self.document_type_handler.detect_document_type(combined_text, total_pages)

        # Try detection strategies in order
        result = None

        # 1. LLM-based detection
        if self.provider and self.provider.is_available():
            try:
                result = self._try_llm_detection(text_chunks, total_pages)
                if result:
                    result.detection_method = "llm"
            except Exception as e:
                logger.warning(f"LLM detection failed: {e}")

        # 2. Content-based detection
        if not result:
            result = self._detect_content_based_boundaries(text_chunks, total_pages)
            if result and result.total_statements > 0:
                result.detection_method = "content_based"

        # 3. Pattern-based fallback
        if not result or result.total_statements == 0:
            result = self.document_type_handler.apply_type_specific_detection(
                document_type, text_chunks, total_pages
            )
            result.detection_method = "pattern_based"

        # Final validation and consolidation
        result.boundaries = self._validate_and_consolidate_boundaries(
            result.boundaries, total_pages
        )
        result.total_statements = len(result.boundaries)
        result.processing_time = time.time() - start_time

        # Cache successful result
        self.cache.set(text_chunks, total_pages, result)

        logger.info(f"Boundary detection completed: {result.total_statements} statements "
                   f"using {result.detection_method} method in {result.processing_time:.2f}s")

        return result

class BoundaryDetectionConfig:
    """Configuration parameters for boundary detection"""

    # LLM Configuration
    LLM_TIMEOUT_SECONDS = 30
    LLM_MAX_RETRIES = 2
    LLM_TEXT_SAMPLE_SIZE = 12000

    # Content Analysis Configuration
    MIN_CONFIDENCE_THRESHOLD = 0.5
    PAGE_MARKER_CONFIDENCE = 0.9
    ACCOUNT_CHANGE_CONFIDENCE = 0.7
    HEADER_DETECTION_CONFIDENCE = 0.6

    # Pattern Matching Configuration
    WESTPAC_12_PAGE_CONFIDENCE = 0.7
    HEURISTIC_CONFIDENCE = 0.6
    DEFAULT_CONFIDENCE = 0.5

    # Validation Configuration
    MAX_STATEMENTS_PER_PAGE = 1.0  # Cannot exceed 1 statement per page
    MIN_STATEMENT_PAGES = 1        # Minimum pages per statement
    MAX_STATEMENT_PAGES = 20       # Maximum reasonable pages per statement

    # Performance Configuration
    CACHE_SIZE = 100
    ENABLE_CACHING = True
    TEXT_OPTIMIZATION_THRESHOLD = 50000  # Characters

class ProviderSpecificConfig:
    """Provider-specific optimization parameters"""

    # OpenAI Configuration
    OPENAI_MODEL = "gpt-4o-mini"
    OPENAI_TEMPERATURE = 0.1
    OPENAI_MAX_TOKENS = 4000
    OPENAI_SYSTEM_PROMPT_VERSION = "v2.1"

    # Ollama Configuration
    OLLAMA_MODEL = "llama3.2"
    OLLAMA_TEMPERATURE = 0.1
    OLLAMA_NUM_PREDICT = 4000
    OLLAMA_TIMEOUT = 60

    # Fallback Configuration
    FALLBACK_METHOD = "content_based"  # or "pattern_based"
    ENABLE_AGGRESSIVE_FALLBACK = True

class BoundaryDetectionMetrics:
    """Metrics collection for boundary detection performance"""

    def __init__(self):
        self.detection_attempts = 0
        self.successful_detections = 0
        self.method_usage = defaultdict(int)
        self.confidence_scores = []
        self.processing_times = []
        self.error_counts = defaultdict(int)

    def record_detection(self, result: BoundaryDetectionResult, error: Optional[Exception] = None):
        """Record boundary detection attempt"""
        self.detection_attempts += 1

        if error:
            self.error_counts[type(error).__name__] += 1
        else:
            self.successful_detections += 1
            self.method_usage[result.detection_method] += 1
            self.processing_times.append(result.processing_time)

            # Record confidence scores
            for boundary in result.boundaries:
                self.confidence_scores.append(boundary.confidence)

    def get_success_rate(self) -> float:
        """Calculate overall success rate"""
        if self.detection_attempts == 0:
            return 0.0
        return self.successful_detections / self.detection_attempts

    def get_method_distribution(self) -> Dict[str, float]:
        """Get distribution of detection methods used"""
        total_successes = sum(self.method_usage.values())
        if total_successes == 0:
            return {}

        return {method: count / total_successes
                for method, count in self.method_usage.items()}

    def get_average_confidence(self) -> float:
        """Calculate average confidence score"""
        if not self.confidence_scores:
            return 0.0
        return sum(self.confidence_scores) / len(self.confidence_scores)

    def get_performance_summary(self) -> Dict[str, Any]:
        """Get comprehensive performance summary"""
        return {
            "success_rate": self.get_success_rate(),
            "total_attempts": self.detection_attempts,
            "method_distribution": self.get_method_distribution(),
            "average_confidence": self.get_average_confidence(),
            "average_processing_time": (sum(self.processing_times) / len(self.processing_times)
                                      if self.processing_times else 0.0),
            "error_distribution": dict(self.error_counts)
        }

def monitor_boundary_detection():
    """Real-time monitoring of boundary detection performance"""

    metrics = BoundaryDetectionMetrics()

    # Set up periodic reporting
    def report_metrics():
        summary = metrics.get_performance_summary()

        logger.info("=== Boundary Detection Performance Summary ===")
        logger.info(f"Success Rate: {summary['success_rate']:.2%}")
        logger.info(f"Average Confidence: {summary['average_confidence']:.3f}")
        logger.info(f"Average Processing Time: {summary['average_processing_time']:.2f}s")
        logger.info(f"Method Distribution: {summary['method_distribution']}")

        if summary['error_distribution']:
            logger.warning(f"Error Distribution: {summary['error_distribution']}")

    # Report every 100 attempts or hourly
    if metrics.detection_attempts % 100 == 0:
        report_metrics()

class AdaptiveBoundaryDetector:
    """ML-enhanced boundary detection"""

    def __init__(self):
        self.feature_extractor = DocumentFeatureExtractor()
        self.boundary_classifier = BoundaryClassificationModel()
        self.confidence_predictor = ConfidencePredictionModel()

    def extract_features(self, text: str, total_pages: int) -> np.ndarray:
        """Extract ML features from document"""
        features = []

        # Text-based features
        features.extend([
            len(text),
            text.count('statement'),
            text.count('account'),
            text.count('balance'),
            len(re.findall(r'\bpage\s+\d+\s+of\s+\d+\b', text, re.IGNORECASE))
        ])

        # Structure-based features
        features.extend([
            total_pages,
            total_pages / len(text) if text else 0,  # Page density
            text.count('\n') / total_pages if total_pages else 0  # Lines per page
        ])

        return np.array(features)

    def predict_boundaries(self, text: str, total_pages: int) -> List[StatementBoundary]:
        """Use ML model to predict boundaries"""
        features = self.extract_features(text, total_pages)

        # Predict boundary positions
        boundary_probabilities = self.boundary_classifier.predict_proba(features)

        # Convert probabilities to boundary objects
        boundaries = []
        for i, prob in enumerate(boundary_probabilities):
            if prob > 0.5:  # Threshold for boundary detection
                confidence = self.confidence_predictor.predict(features)[i]
                boundaries.append(StatementBoundary(
                    start_page=i + 1,
                    end_page=min(i + 5, total_pages),  # Estimated range
                    confidence=confidence,
                    reasoning=f"ML prediction (confidence: {prob:.3f})"
                ))

        return boundaries

class AdvancedPatternRecognition:
    """Advanced pattern recognition for boundary detection"""

    def analyze_document_structure(self, pdf_document) -> DocumentStructure:
        """Analyze PDF structure beyond text content"""

        structure = DocumentStructure()

        # Font analysis
        structure.font_changes = self._detect_font_changes(pdf_document)

        # Layout analysis
        structure.column_layouts = self._analyze_column_layouts(pdf_document)

        # Image/logo detection
        structure.logo_positions = self._detect_logos(pdf_document)

        # Table structure
        structure.table_boundaries = self._detect_tables(pdf_document)

        return structure

    def predict_boundaries_from_structure(self, structure: DocumentStructure) -> List[StatementBoundary]:
        """Predict boundaries based on document structure analysis"""
        boundaries = []

        # Use font changes to detect headers
        for font_change in structure.font_changes:
            if self._is_likely_statement_header(font_change):
                boundaries.append(self._create_boundary_from_font_change(font_change))

        # Use logo positions to detect statement starts
        for logo_pos in structure.logo_positions:
            if self._is_likely_statement_logo(logo_pos):
                boundaries.append(self._create_boundary_from_logo(logo_pos))

        return self._consolidate_structural_boundaries(boundaries)