working version of bulk importer

core/bulk_importer.py

@@ -5,30 +5,65 @@ from docling.document_converter import DocumentConverter
 
 class BulkImporter:
     def __init__(self):
-        # Converter engine for handling multi-column documents
         self.converter = DocumentConverter()
-        # Matches textbook unit codes (e.g., U5_3D, U7 4A, UG_11B)
         self.unit_regex = re.compile(r'U([0-9]+)')
 
-    def clean_text(self, text: str) -> str:
+    def clean_field(self, text: str) -> str:
-        """Cleans syntax breaks, structural commas, and quotes from strings."""
+        """Removes layout spacing, structural artifacts, and noise."""
         if not text:
             return ""
-        return text.replace('\n', ' ').replace('"', '').replace("'", "").strip()
+        # Remove markdown bold/italics markers if any, and compress spaces
+        cleaned = text.replace('*', '').replace('_', '').replace('\\', '')
+        cleaned = re.sub(r'\s+', ' ', cleaned)
+        return cleaned.strip()
 
     def parse_unit(self, context_str: str) -> int:
-        """Extracts the exact Unit integer from codes like U5_3D or U2 1A."""
+        """Extracts the integer unit ID out of strings like 'U3_8B' or 'U7'."""
         match = self.unit_regex.search(context_str)
         if match:
             return int(match.group(1))
-        return None # General reference markers like 'UT LEX' or 'UG'
+        return None
+
+    def insert_pair(self, cursor, es_text, en_text, context_tag, textbook_name):
+        """Helper to cleanly insert a single Spanish-English translation pair."""
+        es_clean = self.clean_field(es_text)
+        en_clean = self.clean_field(en_text)
+        tag_clean = self.clean_field(context_tag)
+        
+        # Guard clause against column headers or empty layout cells
+        if not es_clean or not en_clean:
+            return False
+        if es_clean.lower() in ["spanish", "word", "alphabetical glossary", "es", "en"] or len(es_clean) <= 1:
+            return False
+
+        unit_number = self.parse_unit(tag_clean)
+
+        try:
+            # 1. Insert Spanish Phrase
+            cursor.execute("""
+                INSERT INTO phrases (text, language, textbook, unit, source_context)
+                VALUES (?, 'es', ?, ?, ?)
+            """, (es_clean, textbook_name, unit_number, tag_clean))
+            es_id = cursor.lastrowid
+
+            # 2. Insert English Translation
+            cursor.execute("""
+                INSERT INTO phrases (text, language, textbook, unit, source_context)
+                VALUES (?, 'en', ?, ?, ?)
+            """, (en_clean, textbook_name, unit_number, tag_clean))
+            en_id = cursor.lastrowid
+
+            # 3. Create Bidirectional Cross-Reference Entries
+            cursor.execute("INSERT INTO translations (source_phrase_id, target_phrase_id) VALUES (?, ?)", (es_id, en_id))
+            cursor.execute("INSERT INTO translations (source_phrase_id, target_phrase_id) VALUES (?, ?)", (en_id, es_id))
+            return True
+        except Exception:
+            return False
 
     def import_pdf_glossary(self, file_path: str, textbook_name: str):
-        """Converts multi-column PDF via Docling and maps items to SQLite."""
         print(f"🔄 Docling is analyzing structural layout for '{file_path}'...")
         
         try:
-            # Render layout-aware structural conversion
             result = self.converter.convert(file_path)
             document_text = result.document.export_to_markdown()
         except Exception as e:
@@ -38,52 +73,49 @@ class BulkImporter:
         conn = get_connection()
         cursor = conn.cursor()
         
-        print("📥 Parsing text structures and writing to database...")
+        print("📥 Parsing 6-column text grid structure and writing to database...")
         count = 0
 
-        # Regex targeted to catch both standard blocks and tabular lines safely
+        # Process the markdown line-by-line
-        # Matches patterns like: "Spanish Word", "English Translation", "Unit Code"
+        lines = document_text.split('\n')
-        pattern = re.compile(r'([^,\n"\[]+?)\s*,\s*([^,\n"\[]+?)\s*,\s*(U[G|T|0-9][^\n,]+)')
+        for line in lines:
-        matches = pattern.findall(document_text)
+            line_str = line.strip()
-
+            
-        try:
+            # Target lines containing table formatting row data
-            for es_raw, en_raw, ctx_raw in matches:
+            if not line_str.startswith('|') or not line_str.endswith('|'):
-                spanish_text = self.clean_text(es_raw)
+                continue
-                english_text = self.clean_text(en_raw)
-                context_tag = self.clean_text(ctx_raw)
-                unit_number = self.parse_unit(context_tag)
-
-                # Avoid table headers or non-lexical entries
-                if spanish_text.lower() in ["spanish", "word", "alphabetical glossary"] or len(spanish_text) <= 1:
-                    continue
-
-                # 1. Insert Spanish Term Entry
-                cursor.execute("""
-                    INSERT INTO phrases (text, language, textbook, unit, source_context)
-                    VALUES (?, 'es', ?, ?, ?)
-                """, (spanish_text, textbook_name, unit_number, context_tag))
-                es_id = cursor.lastrowid
-
-                # 2. Insert English Translation Entry
-                cursor.execute("""
-                    INSERT INTO phrases (text, language, textbook, unit, source_context)
-                    VALUES (?, 'en', ?, ?, ?)
-                """, (english_text, textbook_name, unit_number, context_tag))
-                en_id = cursor.lastrowid
-
-                # 3. Create Bidirectional Cross-Reference Records
-                cursor.execute("INSERT INTO translations (source_phrase_id, target_phrase_id) VALUES (?, ?)", (es_id, en_id))
-                cursor.execute("INSERT INTO translations (source_phrase_id, target_phrase_id) VALUES (?, ?)", (en_id, es_id))
                 
+            # Split by markdown table pipes
+            # e.g., "| años60 m | 1960s | U8_7A | bañ osmpl | bath | U1_1A |"
+            parts = [p.strip() for p in line_str.split('|')]
+            
+            # A valid row split will include empty items at the ends due to leading/trailing pipes
+            # For a 6-column table, len(parts) should be at least 8 elements
+            if len(parts) < 7:
+                continue
+                
+            # Skip Markdown table header separator rows: |---|---|...
+            if '---' in parts[1]:
+                continue
+
+            # Extract Left-Hand Column Group (Columns 1, 2, 3)
+            es_left = parts[1]
+            en_left = parts[2]
+            unit_left = parts[3] if len(parts) >= 4 else ""
+            
+            if self.insert_pair(cursor, es_left, en_left, unit_left, textbook_name):
                 count += 1
 
-            conn.commit()
+            # Extract Right-Hand Column Group (Columns 4, 5, 6)
-            print(f"🎉 Bulk ingestion complete! Registered {count} matched glossary items.")
+            if len(parts) >= 7:
-            return count
+                es_right = parts[4]
+                en_right = parts[5]
+                unit_right = parts[6]
+                
+                if self.insert_pair(cursor, es_right, en_right, unit_right, textbook_name):
+                    count += 1
 
-        except Exception as e:
+        conn.commit()
-            conn.rollback()
+        conn.close()
-            print(f"❌ Error during database transaction: {e}")
+        print(f"🎉 Bulk ingestion complete! Registered {count} matched glossary items.")
-            return 0
+        return count
-        finally:
-            conn.close()

doc/Notes.md

@@ -27,6 +27,9 @@
   - [14.1. How edge-tts Works (The Cloud Pipeline)](#141-how-edge-tts-works-the-cloud-pipeline)
   - [14.2. Why This is Highly Advantageous for Phase 1](#142-why-this-is-highly-advantageous-for-phase-1)
   - [14.3. Future Proofing: Going Fully Offline Down the Track](#143-future-proofing-going-fully-offline-down-the-track)
+- [Steps of app creation](#steps-of-app-creation)
+- [Gemini Reposne](#gemini-reposne)
+- [So we fix the bulk upload file](#so-we-fix-the-bulk-upload-file)
 
 # 1. spanish-voice-trainer
 # 2. Project Summary: 
@@ -379,4 +382,32 @@ Kokoro: A highly lightweight, open-source local voice model that produces incred
 
 For now, leveraging Microsoft's cloud via edge-tts gives you native-speaker accuracy with zero setup friction, letting you focus entirely on mastering your Aula Internacional curriculum!
 
-I suppose this OK in phase 1 , I may want to migrate Kokoro next
+I suppose this OK in phase 1 , I may want to migrate Kokoro next
+
+# Steps of app creation
+```zsh
+(139_spanish-voice-trainer) stephenlohning@Scotty 139_spanish-voice-trainer % git tag -n
+list            basic framework created
+v1.0            Tested text speak with 2 files added to the root directory
+v2.0            added db creation, to project, also loaded installed Bee Keeper Studio on Scotty
+v3.0            phase manager added main modified to test this, bulk_importer drafted not tested
+v4.0            bulk importer did not work using docling, so we going to export the markdown into a text file as an intermediate step
+v5.0            modified to export the markdown file, to be used to modify bulk importer
+```
+v5.0 docling gave us
+```txt
+## alphabetical GLOSSARY
+
+| años60 m            | 1960s            | U8_7A       | bañ osmpl                   | bath                    | U1_1A       |
+|---------------------|------------------|-------------|-----------------------------|-------------------------|-------------|
+| aparecer (zc)       | toappear         | U3_6A       | bar m                       | bar                     | U1_LEX      |
+| apasionado/a        | passionate       | U5_15C      | barato/a                    | cheap                   | U4_6C       |
+| apellido m          | surname          | U1_4A       | barba f                     | beard                   | U5_9A       |
+| aprender            | tolearn          | U2_2C       | Barcelona                   | Barcelona               | U3_8A       |
+```
+
+# Gemini Reposne
+```zsh
+| Col 1 (ES) | Col 2 (EN) | Col 3 (Unit) | Col 4 (ES) | Col 5 (EN) | Col 6 (Unit) |
+```
+# So we fix the bulk upload file 

main.py

@@ -1,27 +1,20 @@
 # main.py
 import asyncio
-from docling.document_converter import DocumentConverter
+from database.connection import init_db
+from core.bulk_importer import BulkImporter
 
-async def dump_docling_output():
+async def main():
-    file_path = "aula_int_plus_1_glos_en_alfa.pdf"
+    # Verify tables exist or run schemas
-    output_path = "docling_output.md"
+    init_db()
     
-    print(f"🔄 Docling is analyzing structural layout for '{file_path}'...")
+    # Initialize the importer pipeline
+    importer = BulkImporter()
     
-    try:
+    # Execute structural parse and database write operations
-        converter = DocumentConverter()
+    pdf_file = "aula_int_plus_1_glos_en_alfa.pdf"
-        result = converter.convert(file_path)
+    textbook = "Aula Internacional Plus 1"
-        markdown_text = result.document.export_to_markdown()
+    
-        
+    importer.import_pdf_glossary(pdf_file, textbook)
-        # Save the exact output to a file in your project root
-        with open(output_path, "w", encoding="utf-8") as f:
-            f.write(markdown_text)
-            
-        print(f"📝 Success! Raw Docling layout exported cleanly to: {output_path}")
-        print("👀 Open this file in VS Code to see exactly how the text flows.")
-        
-    except Exception as e:
-        print(f"❌ Docling processing failed: {e}")
 
 if __name__ == "__main__":
-    asyncio.run(dump_docling_output())
+    asyncio.run(main())