139_spanish-voice-trainer/core/bulk_importer.py

# core/bulk_importer.py
import os
import re
from docling.document_converter import DocumentConverter
from database.connection import get_connection

class BulkImporter:
    def import_pdf_glossary(self, pdf_path: str, textbook_name: str):
        """Uses Docling layout extraction engine to parse tables out of multi-column glossary PDFs."""
        print(f"🔄 Analyzing structural layout for '{pdf_path}'...")
        
        if not os.path.exists(pdf_path):
            print(f"❌ Target document path could not be found: {pdf_path}")
            return

        conn = get_connection()
        cursor = conn.cursor()
        
        # Reset staging tables to guarantee fresh data state
        cursor.execute("DELETE FROM translations")
        cursor.execute("DELETE FROM phrases")
        conn.commit()
        
        print("📥 Initializing Docling layout analysis engine...")
        try:
            # 1. Initialize Docling converter
            converter = DocumentConverter()
            result = converter.convert(pdf_path)
            
            print("📥 Parsing extracted text tables and tabular structures...")
            raw_inserts_count = 0

            # 2. Iterate through extracted tables found inside the layout structure
            for table_idx, table_element in enumerate(result.document.tables):
                # Convert Docling table data framework back to standard pandas-like dictionary arrays
                table_data = table_element.export_to_dataframe()
                
                # Iterate rows while ensuring it's not looking at headers or broken table pieces
                for row_idx, row in table_data.iterrows():
                    row_list = list(row)
                    
                    # Ensure we have enough columns to look at your glossary structure (expecting 4-6 columns)
                    if len(row_list) < 3:
                        continue
                        
                    # Extract positions out of the 6-column structure:
                    # Usually: Col 0 = Spanish, Col 1 = English, Col 2 or 3 = Context/Unit tag
                    es_raw = str(row_list[0]).strip() if row_list[0] else ""
                    en_raw = str(row_list[1]).strip() if row_list[1] else ""
                    context_raw = str(row_list[2]).strip() if row_list[2] else ""
                    
                    # Ignore empty lines or column labels (like "Spanish", "Español", "English")
                    if not es_raw or not en_raw or "español" in es_raw.lower() or "english" in en_raw.lower():
                        continue
                    
                    # Parse out unit identifiers from context strings (e.g. "U2_3A" gives unit = 2)
                    unit_num = 1
                    unit_match = re.search(r'U(\d+)', context_raw, re.IGNORECASE)
                    if unit_match:
                        unit_num = int(unit_match.group(1))

                    # 3. Stage Spanish item row
                    cursor.execute("""
                        INSERT INTO phrases (text, language, textbook, unit, source_context, word_type)
                        VALUES (?, 'es', ?, ?, ?, 'phrase')
                    """, (es_raw, textbook_name, unit_num, context_raw))
                    
                    # 4. Stage English item row (immediately adjacent)
                    cursor.execute("""
                        INSERT INTO phrases (text, language, textbook, unit, source_context, word_type)
                        VALUES (?, 'en', ?, ?, ?, 'phrase')
                    """, (en_raw, textbook_name, unit_num, context_raw))
                    
                    raw_inserts_count += 2

            conn.commit()
            print(f"🎉 Bulk ingestion staging complete! Staged {raw_inserts_count} entries.")
            
        except Exception as docling_error:
            print(f"❌ Docling encountered a processing failure: {docling_error}")
            import traceback
            traceback.print_exc()
        finally:
            conn.close()