# core/bulk_importer.py import re from database.connection import get_connection class BulkImporter: def __init__(self): # Matches typical patterns: "Spanish word", "English translation", "Unit metadata" self.row_regex = re.compile(r'"([^"]+)"\s*,\s*"([^"]+)"\s*,\s*"([^"]+)"') def clean_text(self, text: str) -> str: """Strips newlines and extra spaces from extracted data fields.""" return text.replace('\n', ' ').strip() def parse_unit(self, context_str: str) -> int: """ Extracts the unit integer from codes like 'U5_3D', 'U7 4A', or 'UG_11B'. Returns None if it's a general marker like 'UT LEX'. """ match = re.search(r'U([0-9])', context_str) if match: return int(match.group(1)) return None def import_glossary_file(self, file_path: str, textbook_name: str): """Reads the structural text lines and inserts them into the translation database.""" conn = get_connection() cursor = conn.cursor() print(f"📖 Starting bulk ingestion for '{file_path}'...") count = 0 try: with open(file_path, 'r', encoding='utf-8') as f: content = f.read() # Find all matching row structures inside the text matches = self.row_regex.findall(content) for es_raw, en_raw, ctx_raw in matches: spanish_text = self.clean_text(es_raw) english_text = self.clean_text(en_raw) context_tag = self.clean_text(ctx_raw) unit_number = self.parse_unit(context_tag) # Skip header rows or structural markers if spanish_text.lower() in ["alphabetical glossary", "spanish", "word"]: continue # 1. Insert Spanish Term cursor.execute(""" INSERT INTO phrases (text, language, textbook, unit, source_context) VALUES (?, 'es', ?, ?, ?) """, (spanish_text, textbook_name, unit_number, context_tag)) es_id = cursor.lastrowid # 2. Insert English Term cursor.execute(""" INSERT INTO phrases (text, language, textbook, unit, source_context) VALUES (?, 'en', ?, ?, ?) """, (english_text, textbook_name, unit_number, context_tag)) en_id = cursor.lastrowid # 3. Create Bidirectional Bridges cursor.execute("INSERT INTO translations (source_phrase_id, target_phrase_id) VALUES (?, ?)", (es_id, en_id)) cursor.execute("INSERT INTO translations (source_phrase_id, target_phrase_id) VALUES (?, ?)", (en_id, es_id)) count += 1 conn.commit() print(f"🎉 Successfully imported {count} linked glossary pairs into SQLite!") return count except Exception as e: conn.rollback() print(f"❌ Error during bulk data ingestion: {e}") return 0 finally: conn.close()