112 lines
No EOL
4.9 KiB
Python
112 lines
No EOL
4.9 KiB
Python
# core/clean_glossary.py
|
|
import re
|
|
from database.connection import get_connection
|
|
|
|
class GlossaryCleaner:
|
|
def __init__(self):
|
|
self.verb_pattern = re.compile(r'\s*\(([a-z\s]+)\)')
|
|
|
|
def clean_and_expand_spanish(self, text: str) -> list[tuple[str, str, str]]:
|
|
word_type = "phrase"
|
|
grammar_note = None
|
|
clean_text = str(text).strip()
|
|
|
|
verb_match = self.verb_pattern.search(clean_text)
|
|
if verb_match:
|
|
word_type = "verb"
|
|
grammar_note = verb_match.group(1).strip()
|
|
clean_text = self.verb_pattern.sub('', clean_text).strip()
|
|
|
|
lower_text = clean_text.lower()
|
|
if lower_text.endswith(('mpl', 'fpl', 'smpl', 'osmpl')):
|
|
word_type = "noun"
|
|
grammar_note = "f, pl" if 'f' in lower_text else "m, pl"
|
|
if lower_text.endswith('osmpl'):
|
|
clean_text = clean_text[:-5] + "os"
|
|
else:
|
|
clean_text = re.sub(r'\s*[a-zA-Z\s]*$', '', clean_text)
|
|
elif re.search(r'\s+\b(m|f|pl|m,\s*pl|f,\s*pl)\b\s*$', clean_text):
|
|
word_type = "noun"
|
|
match = re.search(r'\b(m|f|pl|m,\s*pl|f,\s*pl)\b\s*$', clean_text)
|
|
if match:
|
|
grammar_note = match.group(1).strip()
|
|
clean_text = re.sub(r'\s+\b(m|f|pl|m,\s*pl|f,\s*pl)\b\s*$', '', clean_text).strip()
|
|
|
|
return [(clean_text.strip(), word_type, grammar_note)]
|
|
|
|
def clean_english_text(self, text: str) -> str:
|
|
cleaned = str(text).strip()
|
|
cleaned = re.sub(r'\s+', ' ', cleaned)
|
|
if cleaned.startswith("to") and len(cleaned) > 2 and not cleaned.startswith("to "):
|
|
cleaned = re.sub(r'^to([a-z])', r'to \1', cleaned)
|
|
return cleaned
|
|
|
|
def process_database_clean(self):
|
|
"""Processes raw entries from the source tables and builds translation row connections."""
|
|
conn = get_connection()
|
|
cursor = conn.cursor()
|
|
|
|
# Pull raw rows populated during BulkImporter phase
|
|
# Ensuring we look at your base uncleaned layout records
|
|
try:
|
|
cursor.execute("SELECT id, text, language, textbook, unit, source_context FROM phrases")
|
|
raw_rows = cursor.fetchall()
|
|
except Exception as e:
|
|
print(f"⚠️ Error reading raw phrases: {e}")
|
|
conn.close()
|
|
return
|
|
|
|
# Separate them logically to reconstruct pairing matrices safely
|
|
# We find adjacent lines assuming index N is Spanish and N+1 is its English partner
|
|
paired_rows = []
|
|
for i in range(0, len(raw_rows) - 1, 2):
|
|
if raw_rows[i][2] == 'es':
|
|
paired_rows.append((
|
|
raw_rows[i][1], # Spanish raw text
|
|
raw_rows[i+1][1], # English raw text
|
|
raw_rows[i][3], # Textbook name
|
|
raw_rows[i][4], # Unit Integer
|
|
raw_rows[i][5] # Context string (e.g. U2_3A)
|
|
))
|
|
|
|
# Clear down active runtime tables to map fresh structural pairs
|
|
cursor.execute("DELETE FROM translations")
|
|
cursor.execute("DELETE FROM phrases")
|
|
|
|
print(f"⚙️ Migrating {len(paired_rows)} raw rows into translation pairs...")
|
|
inserted_count = 0
|
|
|
|
for es_raw, en_raw, textbook, unit, context in paired_rows:
|
|
# Explicitly keep underscore formatting (e.g., U2_3A) intact
|
|
safe_context = str(context).strip() if context else f"U{unit}"
|
|
|
|
# Clean both sides through individual language filters
|
|
es_variants = self.clean_and_expand_spanish(es_raw)
|
|
en_clean = self.clean_english_text(en_raw)
|
|
|
|
for es_clean, w_type, g_note in es_variants:
|
|
# 1. Store Spanish Text Item Node
|
|
cursor.execute("""
|
|
INSERT INTO phrases (text, language, textbook, unit, source_context, word_type, grammar_note)
|
|
VALUES (?, 'es', ?, ?, ?, ?, ?)
|
|
""", (es_clean, textbook, unit, safe_context, w_type, g_note))
|
|
es_id = cursor.lastrowid
|
|
|
|
# 2. Store English Text Item Node
|
|
cursor.execute("""
|
|
INSERT INTO phrases (text, language, textbook, unit, source_context, word_type, grammar_note)
|
|
VALUES (?, 'en', ?, ?, ?, ?, NULL)
|
|
""", (en_clean, textbook, unit, safe_context, w_type))
|
|
en_id = cursor.lastrowid
|
|
|
|
# 3. Form unique single translation row mapping bond
|
|
cursor.execute("""
|
|
INSERT INTO translations (source_phrase_id, target_phrase_id, deck_name)
|
|
VALUES (?, ?, 'General')
|
|
""", (es_id, en_id))
|
|
|
|
inserted_count += 1
|
|
|
|
conn.commit()
|
|
conn.close()
|
|
print(f"🎉 Schema extraction pipeline completed! Saved {inserted_count} pristine translation pairs.") |