78 lines
3.1 KiB
Python
78 lines
3.1 KiB
Python
|
# core/bulk_importer.py
|
||
|
|
import re
|
||
|
|
from database.connection import get_connection
|
||
|
|
|
||
|
|
class BulkImporter:
|
||
|
|
def __init__(self):
|
||
|
|
# Matches typical patterns: "Spanish word", "English translation", "Unit metadata"
|
||
|
|
self.row_regex = re.compile(r'"([^"]+)"\s*,\s*"([^"]+)"\s*,\s*"([^"]+)"')
|
||
|
|
|
||
|
|
def clean_text(self, text: str) -> str:
|
||
|
|
"""Strips newlines and extra spaces from extracted data fields."""
|
||
|
|
return text.replace('\n', ' ').strip()
|
||
|
|
|
||
|
|
def parse_unit(self, context_str: str) -> int:
|
||
|
|
"""
|
||
|
|
Extracts the unit integer from codes like 'U5_3D', 'U7 4A', or 'UG_11B'.
|
||
|
|
Returns None if it's a general marker like 'UT LEX'.
|
||
|
|
"""
|
||
|
|
match = re.search(r'U([0-9])', context_str)
|
||
|
|
if match:
|
||
|
|
return int(match.group(1))
|
||
|
|
return None
|
||
|
|
|
||
|
|
def import_glossary_file(self, file_path: str, textbook_name: str):
|
||
|
|
"""Reads the structural text lines and inserts them into the translation database."""
|
||
|
|
conn = get_connection()
|
||
|
|
cursor = conn.cursor()
|
||
|
|
|
||
|
|
print(f"📖 Starting bulk ingestion for '{file_path}'...")
|
||
|
|
count = 0
|
||
|
|
|
||
|
|
try:
|
||
|
|
with open(file_path, 'r', encoding='utf-8') as f:
|
||
|
|
content = f.read()
|
||
|
|
|
||
|
|
# Find all matching row structures inside the text
|
||
|
|
matches = self.row_regex.findall(content)
|
||
|
|
|
||
|
|
for es_raw, en_raw, ctx_raw in matches:
|
||
|
|
spanish_text = self.clean_text(es_raw)
|
||
|
|
english_text = self.clean_text(en_raw)
|
||
|
|
context_tag = self.clean_text(ctx_raw)
|
||
|
|
unit_number = self.parse_unit(context_tag)
|
||
|
|
|
||
|
|
# Skip header rows or structural markers
|
||
|
|
if spanish_text.lower() in ["alphabetical glossary", "spanish", "word"]:
|
||
|
|
continue
|
||
|
|
|
||
|
|
# 1. Insert Spanish Term
|
||
|
|
cursor.execute("""
|
||
|
|
INSERT INTO phrases (text, language, textbook, unit, source_context)
|
||
|
|
VALUES (?, 'es', ?, ?, ?)
|
||
|
|
""", (spanish_text, textbook_name, unit_number, context_tag))
|
||
|
|
es_id = cursor.lastrowid
|
||
|
|
|
||
|
|
# 2. Insert English Term
|
||
|
|
cursor.execute("""
|
||
|
|
INSERT INTO phrases (text, language, textbook, unit, source_context)
|
||
|
|
VALUES (?, 'en', ?, ?, ?)
|
||
|
|
""", (english_text, textbook_name, unit_number, context_tag))
|
||
|
|
en_id = cursor.lastrowid
|
||
|
|
|
||
|
|
# 3. Create Bidirectional Bridges
|
||
|
|
cursor.execute("INSERT INTO translations (source_phrase_id, target_phrase_id) VALUES (?, ?)", (es_id, en_id))
|
||
|
|
cursor.execute("INSERT INTO translations (source_phrase_id, target_phrase_id) VALUES (?, ?)", (en_id, es_id))
|
||
|
|
|
||
|
|
count += 1
|
||
|
|
|
||
|
|
conn.commit()
|
||
|
|
print(f"🎉 Successfully imported {count} linked glossary pairs into SQLite!")
|
||
|
|
return count
|
||
|
|
|
||
|
|
except Exception as e:
|
||
|
|
conn.rollback()
|
||
|
|
print(f"❌ Error during bulk data ingestion: {e}")
|
||
|
|
return 0
|
||
|
|
finally:
|
||
|
|
conn.close()
|