bulk importer did not work using docling

core/bulk_importer.py

@@ -1,78 +1,89 @@
 # core/bulk_importer.py
 import re
 from database.connection import get_connection
+from docling.document_converter import DocumentConverter
 
 class BulkImporter:
     def __init__(self):
-        # Matches typical patterns: "Spanish word", "English translation", "Unit metadata"
-        self.row_regex = re.compile(r'"([^"]+)"\s*,\s*"([^"]+)"\s*,\s*"([^"]+)"')
+        # Converter engine for handling multi-column documents
+        self.converter = DocumentConverter()
+        # Matches textbook unit codes (e.g., U5_3D, U7 4A, UG_11B)
+        self.unit_regex = re.compile(r'U([0-9]+)')
 
     def clean_text(self, text: str) -> str:
-        """Strips newlines and extra spaces from extracted data fields."""
-        return text.replace('\n', ' ').strip()
+        """Cleans syntax breaks, structural commas, and quotes from strings."""
+        if not text:
+            return ""
+        return text.replace('\n', ' ').replace('"', '').replace("'", "").strip()
 
     def parse_unit(self, context_str: str) -> int:
-        """
-        Extracts the unit integer from codes like 'U5_3D', 'U7 4A', or 'UG_11B'.
-        Returns None if it's a general marker like 'UT LEX'.
-        """
-        match = re.search(r'U([0-9])', context_str)
+        """Extracts the exact Unit integer from codes like U5_3D or U2 1A."""
+        match = self.unit_regex.search(context_str)
         if match:
             return int(match.group(1))
-        return None
+        return None # General reference markers like 'UT LEX' or 'UG'
+
+    def import_pdf_glossary(self, file_path: str, textbook_name: str):
+        """Converts multi-column PDF via Docling and maps items to SQLite."""
+        print(f"🔄 Docling is analyzing structural layout for '{file_path}'...")
+        
+        try:
+            # Render layout-aware structural conversion
+            result = self.converter.convert(file_path)
+            document_text = result.document.export_to_markdown()
+        except Exception as e:
+            print(f"❌ Docling processing failed: {e}")
+            return 0
 
-    def import_glossary_file(self, file_path: str, textbook_name: str):
-        """Reads the structural text lines and inserts them into the translation database."""
         conn = get_connection()
         cursor = conn.cursor()
         
-        print(f"📖 Starting bulk ingestion for '{file_path}'...")
+        print("📥 Parsing text structures and writing to database...")
         count = 0
-        
+
+        # Regex targeted to catch both standard blocks and tabular lines safely
+        # Matches patterns like: "Spanish Word", "English Translation", "Unit Code"
+        pattern = re.compile(r'([^,\n"\[]+?)\s*,\s*([^,\n"\[]+?)\s*,\s*(U[G|T|0-9][^\n,]+)')
+        matches = pattern.findall(document_text)
+
         try:
-            with open(file_path, 'r', encoding='utf-8') as f:
-                content = f.read()
-            
-            # Find all matching row structures inside the text
-            matches = self.row_regex.findall(content)
-            
             for es_raw, en_raw, ctx_raw in matches:
                 spanish_text = self.clean_text(es_raw)
                 english_text = self.clean_text(en_raw)
                 context_tag = self.clean_text(ctx_raw)
                 unit_number = self.parse_unit(context_tag)
-                
-                # Skip header rows or structural markers
-                if spanish_text.lower() in ["alphabetical glossary", "spanish", "word"]:
+
+                # Avoid table headers or non-lexical entries
+                if spanish_text.lower() in ["spanish", "word", "alphabetical glossary"] or len(spanish_text) <= 1:
                     continue
 
-                # 1. Insert Spanish Term
+                # 1. Insert Spanish Term Entry
                 cursor.execute("""
                     INSERT INTO phrases (text, language, textbook, unit, source_context)
                     VALUES (?, 'es', ?, ?, ?)
                 """, (spanish_text, textbook_name, unit_number, context_tag))
                 es_id = cursor.lastrowid
 
-                # 2. Insert English Term
+                # 2. Insert English Translation Entry
                 cursor.execute("""
                     INSERT INTO phrases (text, language, textbook, unit, source_context)
                     VALUES (?, 'en', ?, ?, ?)
                 """, (english_text, textbook_name, unit_number, context_tag))
                 en_id = cursor.lastrowid
 
-                # 3. Create Bidirectional Bridges
+                # 3. Create Bidirectional Cross-Reference Records
                 cursor.execute("INSERT INTO translations (source_phrase_id, target_phrase_id) VALUES (?, ?)", (es_id, en_id))
                 cursor.execute("INSERT INTO translations (source_phrase_id, target_phrase_id) VALUES (?, ?)", (en_id, es_id))
                 
                 count += 1
 
             conn.commit()
-            print(f"🎉 Successfully imported {count} linked glossary pairs into SQLite!")
+            print(f"🎉 Bulk ingestion complete! Registered {count} matched glossary items.")
             return count
 
         except Exception as e:
             conn.rollback()
-            print(f"❌ Error during bulk data ingestion: {e}")
+            print(f"❌ Error during database transaction: {e}")
             return 0
         finally:
             conn.close()

main.py

@@ -1,33 +1,20 @@
 # main.py
-import sys
 import asyncio
 from database.connection import init_db
-from core.phrase_manager import PhraseManager
+from core.bulk_importer import BulkImporter
 
-async def test_pipeline():
-    print("🚀 Booting Castilian Voice Trainer Ingestion Engine...")
-    
-    # Ensure database is present
+async def main():
+    # Keep database schemas active and verified
     init_db()
     
-    # Initialize the phrase controller manager
-    manager = PhraseManager()
+    # Initialize our ingestion utility
+    importer = BulkImporter()
     
-    # Test Entry: Let's log an authentic textbook phrase from Aula Internacional
-    print("\n📥 Processing sample entry...")
-    success = await manager.add_translation_pair(
-        spanish_text="¿Cómo se pronuncia esta palabra?",
-        english_text="How do you pronounce this word?",
-        textbook="Aula Internacional 1 Plus",
-        unit=2,
-        context="Glossary / Lesson Terms",
-        voice_gender="female"  # Let's verify Elvira's voice on this one!
+    # Process the file sitting right in your root directory
+    importer.import_pdf_glossary(
+        file_path="aula_int_plus_1_glos_en_alfa.pdf",
+        textbook_name="Aula Internacional 1 Plus"
     )
-    
-    if success:
-        print("\n🎉 Verification Phase 1 Pipeline Test complete!")
-    else:
-        print("\n⚠️ Pipeline processing failed.")
 
 if __name__ == "__main__":
-    asyncio.run(test_pipeline())
+    asyncio.run(main())

pyproject.toml

@@ -5,6 +5,7 @@ description = "Add your description here"
 readme = "README.md"
 requires-python = ">=3.13"
 dependencies = [
+    "docling>=2.102.1",
     "edge-tts>=7.2.8",
     "fastdtw>=0.3.4",
     "genanki>=0.13.1",