diff --git a/core/bulk_importer.py b/core/bulk_importer.py index 71d37a8..a6bd2bf 100644 --- a/core/bulk_importer.py +++ b/core/bulk_importer.py @@ -5,30 +5,65 @@ from docling.document_converter import DocumentConverter class BulkImporter: def __init__(self): - # Converter engine for handling multi-column documents self.converter = DocumentConverter() - # Matches textbook unit codes (e.g., U5_3D, U7 4A, UG_11B) self.unit_regex = re.compile(r'U([0-9]+)') - def clean_text(self, text: str) -> str: - """Cleans syntax breaks, structural commas, and quotes from strings.""" + def clean_field(self, text: str) -> str: + """Removes layout spacing, structural artifacts, and noise.""" if not text: return "" - return text.replace('\n', ' ').replace('"', '').replace("'", "").strip() + # Remove markdown bold/italics markers if any, and compress spaces + cleaned = text.replace('*', '').replace('_', '').replace('\\', '') + cleaned = re.sub(r'\s+', ' ', cleaned) + return cleaned.strip() def parse_unit(self, context_str: str) -> int: - """Extracts the exact Unit integer from codes like U5_3D or U2 1A.""" + """Extracts the integer unit ID out of strings like 'U3_8B' or 'U7'.""" match = self.unit_regex.search(context_str) if match: return int(match.group(1)) - return None # General reference markers like 'UT LEX' or 'UG' + return None + + def insert_pair(self, cursor, es_text, en_text, context_tag, textbook_name): + """Helper to cleanly insert a single Spanish-English translation pair.""" + es_clean = self.clean_field(es_text) + en_clean = self.clean_field(en_text) + tag_clean = self.clean_field(context_tag) + + # Guard clause against column headers or empty layout cells + if not es_clean or not en_clean: + return False + if es_clean.lower() in ["spanish", "word", "alphabetical glossary", "es", "en"] or len(es_clean) <= 1: + return False + + unit_number = self.parse_unit(tag_clean) + + try: + # 1. Insert Spanish Phrase + cursor.execute(""" + INSERT INTO phrases (text, language, textbook, unit, source_context) + VALUES (?, 'es', ?, ?, ?) + """, (es_clean, textbook_name, unit_number, tag_clean)) + es_id = cursor.lastrowid + + # 2. Insert English Translation + cursor.execute(""" + INSERT INTO phrases (text, language, textbook, unit, source_context) + VALUES (?, 'en', ?, ?, ?) + """, (en_clean, textbook_name, unit_number, tag_clean)) + en_id = cursor.lastrowid + + # 3. Create Bidirectional Cross-Reference Entries + cursor.execute("INSERT INTO translations (source_phrase_id, target_phrase_id) VALUES (?, ?)", (es_id, en_id)) + cursor.execute("INSERT INTO translations (source_phrase_id, target_phrase_id) VALUES (?, ?)", (en_id, es_id)) + return True + except Exception: + return False def import_pdf_glossary(self, file_path: str, textbook_name: str): - """Converts multi-column PDF via Docling and maps items to SQLite.""" print(f"🔄 Docling is analyzing structural layout for '{file_path}'...") try: - # Render layout-aware structural conversion result = self.converter.convert(file_path) document_text = result.document.export_to_markdown() except Exception as e: @@ -38,52 +73,49 @@ class BulkImporter: conn = get_connection() cursor = conn.cursor() - print("📥 Parsing text structures and writing to database...") + print("📥 Parsing 6-column text grid structure and writing to database...") count = 0 - # Regex targeted to catch both standard blocks and tabular lines safely - # Matches patterns like: "Spanish Word", "English Translation", "Unit Code" - pattern = re.compile(r'([^,\n"\[]+?)\s*,\s*([^,\n"\[]+?)\s*,\s*(U[G|T|0-9][^\n,]+)') - matches = pattern.findall(document_text) - - try: - for es_raw, en_raw, ctx_raw in matches: - spanish_text = self.clean_text(es_raw) - english_text = self.clean_text(en_raw) - context_tag = self.clean_text(ctx_raw) - unit_number = self.parse_unit(context_tag) - - # Avoid table headers or non-lexical entries - if spanish_text.lower() in ["spanish", "word", "alphabetical glossary"] or len(spanish_text) <= 1: - continue - - # 1. Insert Spanish Term Entry - cursor.execute(""" - INSERT INTO phrases (text, language, textbook, unit, source_context) - VALUES (?, 'es', ?, ?, ?) - """, (spanish_text, textbook_name, unit_number, context_tag)) - es_id = cursor.lastrowid - - # 2. Insert English Translation Entry - cursor.execute(""" - INSERT INTO phrases (text, language, textbook, unit, source_context) - VALUES (?, 'en', ?, ?, ?) - """, (english_text, textbook_name, unit_number, context_tag)) - en_id = cursor.lastrowid - - # 3. Create Bidirectional Cross-Reference Records - cursor.execute("INSERT INTO translations (source_phrase_id, target_phrase_id) VALUES (?, ?)", (es_id, en_id)) - cursor.execute("INSERT INTO translations (source_phrase_id, target_phrase_id) VALUES (?, ?)", (en_id, es_id)) + # Process the markdown line-by-line + lines = document_text.split('\n') + for line in lines: + line_str = line.strip() + + # Target lines containing table formatting row data + if not line_str.startswith('|') or not line_str.endswith('|'): + continue + # Split by markdown table pipes + # e.g., "| años60 m | 1960s | U8_7A | bañ osmpl | bath | U1_1A |" + parts = [p.strip() for p in line_str.split('|')] + + # A valid row split will include empty items at the ends due to leading/trailing pipes + # For a 6-column table, len(parts) should be at least 8 elements + if len(parts) < 7: + continue + + # Skip Markdown table header separator rows: |---|---|... + if '---' in parts[1]: + continue + + # Extract Left-Hand Column Group (Columns 1, 2, 3) + es_left = parts[1] + en_left = parts[2] + unit_left = parts[3] if len(parts) >= 4 else "" + + if self.insert_pair(cursor, es_left, en_left, unit_left, textbook_name): count += 1 - conn.commit() - print(f"🎉 Bulk ingestion complete! Registered {count} matched glossary items.") - return count + # Extract Right-Hand Column Group (Columns 4, 5, 6) + if len(parts) >= 7: + es_right = parts[4] + en_right = parts[5] + unit_right = parts[6] + + if self.insert_pair(cursor, es_right, en_right, unit_right, textbook_name): + count += 1 - except Exception as e: - conn.rollback() - print(f"❌ Error during database transaction: {e}") - return 0 - finally: - conn.close() \ No newline at end of file + conn.commit() + conn.close() + print(f"🎉 Bulk ingestion complete! Registered {count} matched glossary items.") + return count \ No newline at end of file diff --git a/doc/Notes.md b/doc/Notes.md index 2894edb..4dd83ae 100644 --- a/doc/Notes.md +++ b/doc/Notes.md @@ -27,6 +27,9 @@ - [14.1. How edge-tts Works (The Cloud Pipeline)](#141-how-edge-tts-works-the-cloud-pipeline) - [14.2. Why This is Highly Advantageous for Phase 1](#142-why-this-is-highly-advantageous-for-phase-1) - [14.3. Future Proofing: Going Fully Offline Down the Track](#143-future-proofing-going-fully-offline-down-the-track) +- [Steps of app creation](#steps-of-app-creation) +- [Gemini Reposne](#gemini-reposne) +- [So we fix the bulk upload file](#so-we-fix-the-bulk-upload-file) # 1. spanish-voice-trainer # 2. Project Summary: @@ -379,4 +382,32 @@ Kokoro: A highly lightweight, open-source local voice model that produces incred For now, leveraging Microsoft's cloud via edge-tts gives you native-speaker accuracy with zero setup friction, letting you focus entirely on mastering your Aula Internacional curriculum! -I suppose this OK in phase 1 , I may want to migrate Kokoro next \ No newline at end of file +I suppose this OK in phase 1 , I may want to migrate Kokoro next + +# Steps of app creation +```zsh +(139_spanish-voice-trainer) stephenlohning@Scotty 139_spanish-voice-trainer % git tag -n +list basic framework created +v1.0 Tested text speak with 2 files added to the root directory +v2.0 added db creation, to project, also loaded installed Bee Keeper Studio on Scotty +v3.0 phase manager added main modified to test this, bulk_importer drafted not tested +v4.0 bulk importer did not work using docling, so we going to export the markdown into a text file as an intermediate step +v5.0 modified to export the markdown file, to be used to modify bulk importer +``` +v5.0 docling gave us +```txt +## alphabetical GLOSSARY + +| años60 m | 1960s | U8_7A | bañ osmpl | bath | U1_1A | +|---------------------|------------------|-------------|-----------------------------|-------------------------|-------------| +| aparecer (zc) | toappear | U3_6A | bar m | bar | U1_LEX | +| apasionado/a | passionate | U5_15C | barato/a | cheap | U4_6C | +| apellido m | surname | U1_4A | barba f | beard | U5_9A | +| aprender | tolearn | U2_2C | Barcelona | Barcelona | U3_8A | +``` + +# Gemini Reposne +```zsh +| Col 1 (ES) | Col 2 (EN) | Col 3 (Unit) | Col 4 (ES) | Col 5 (EN) | Col 6 (Unit) | +``` +# So we fix the bulk upload file diff --git a/doc/Notes.pdf b/doc/Notes.pdf index fec0d7d..3bc504f 100644 Binary files a/doc/Notes.pdf and b/doc/Notes.pdf differ diff --git a/main.py b/main.py index da25661..d60e3ed 100644 --- a/main.py +++ b/main.py @@ -1,27 +1,20 @@ # main.py import asyncio -from docling.document_converter import DocumentConverter +from database.connection import init_db +from core.bulk_importer import BulkImporter -async def dump_docling_output(): - file_path = "aula_int_plus_1_glos_en_alfa.pdf" - output_path = "docling_output.md" +async def main(): + # Verify tables exist or run schemas + init_db() - print(f"🔄 Docling is analyzing structural layout for '{file_path}'...") + # Initialize the importer pipeline + importer = BulkImporter() - try: - converter = DocumentConverter() - result = converter.convert(file_path) - markdown_text = result.document.export_to_markdown() - - # Save the exact output to a file in your project root - with open(output_path, "w", encoding="utf-8") as f: - f.write(markdown_text) - - print(f"📝 Success! Raw Docling layout exported cleanly to: {output_path}") - print("👀 Open this file in VS Code to see exactly how the text flows.") - - except Exception as e: - print(f"❌ Docling processing failed: {e}") + # Execute structural parse and database write operations + pdf_file = "aula_int_plus_1_glos_en_alfa.pdf" + textbook = "Aula Internacional Plus 1" + + importer.import_pdf_glossary(pdf_file, textbook) if __name__ == "__main__": - asyncio.run(dump_docling_output()) \ No newline at end of file + asyncio.run(main()) \ No newline at end of file