# core/clean_glossary.py import re from database.connection import get_connection class GlossaryCleaner: def __init__(self): self.verb_pattern = re.compile(r'\s*\(([a-z\s]+)\)') def clean_and_expand_spanish(self, text: str) -> list[tuple[str, str, str]]: word_type = "phrase" grammar_note = None clean_text = str(text).strip() verb_match = self.verb_pattern.search(clean_text) if verb_match: word_type = "verb" grammar_note = verb_match.group(1).strip() clean_text = self.verb_pattern.sub('', clean_text).strip() lower_text = clean_text.lower() if lower_text.endswith(('mpl', 'fpl', 'smpl', 'osmpl')): word_type = "noun" grammar_note = "f, pl" if 'f' in lower_text else "m, pl" if lower_text.endswith('osmpl'): clean_text = clean_text[:-5] + "os" else: clean_text = re.sub(r'\s*[a-zA-Z\s]*$', '', clean_text) elif re.search(r'\s+\b(m|f|pl|m,\s*pl|f,\s*pl)\b\s*$', clean_text): word_type = "noun" match = re.search(r'\b(m|f|pl|m,\s*pl|f,\s*pl)\b\s*$', clean_text) if match: grammar_note = match.group(1).strip() clean_text = re.sub(r'\s+\b(m|f|pl|m,\s*pl|f,\s*pl)\b\s*$', '', clean_text).strip() return [(clean_text.strip(), word_type, grammar_note)] def clean_english_text(self, text: str) -> str: cleaned = str(text).strip() cleaned = re.sub(r'\s+', ' ', cleaned) if cleaned.startswith("to") and len(cleaned) > 2 and not cleaned.startswith("to "): cleaned = re.sub(r'^to([a-z])', r'to \1', cleaned) return cleaned def process_database_clean(self): """Processes raw entries from the source tables and builds translation row connections.""" conn = get_connection() cursor = conn.cursor() # Pull raw rows populated during BulkImporter phase # Ensuring we look at your base uncleaned layout records try: cursor.execute("SELECT id, text, language, textbook, unit, source_context FROM phrases") raw_rows = cursor.fetchall() except Exception as e: print(f"⚠️ Error reading raw phrases: {e}") conn.close() return # Separate them logically to reconstruct pairing matrices safely # We find adjacent lines assuming index N is Spanish and N+1 is its English partner paired_rows = [] for i in range(0, len(raw_rows) - 1, 2): if raw_rows[i][2] == 'es': paired_rows.append(( raw_rows[i][1], # Spanish raw text raw_rows[i+1][1], # English raw text raw_rows[i][3], # Textbook name raw_rows[i][4], # Unit Integer raw_rows[i][5] # Context string (e.g. U2_3A) )) # Clear down active runtime tables to map fresh structural pairs cursor.execute("DELETE FROM translations") cursor.execute("DELETE FROM phrases") print(f"⚙️ Migrating {len(paired_rows)} raw rows into translation pairs...") inserted_count = 0 for es_raw, en_raw, textbook, unit, context in paired_rows: # Explicitly keep underscore formatting (e.g., U2_3A) intact safe_context = str(context).strip() if context else f"U{unit}" # Clean both sides through individual language filters es_variants = self.clean_and_expand_spanish(es_raw) en_clean = self.clean_english_text(en_raw) for es_clean, w_type, g_note in es_variants: # 1. Store Spanish Text Item Node cursor.execute(""" INSERT INTO phrases (text, language, textbook, unit, source_context, word_type, grammar_note) VALUES (?, 'es', ?, ?, ?, ?, ?) """, (es_clean, textbook, unit, safe_context, w_type, g_note)) es_id = cursor.lastrowid # 2. Store English Text Item Node cursor.execute(""" INSERT INTO phrases (text, language, textbook, unit, source_context, word_type, grammar_note) VALUES (?, 'en', ?, ?, ?, ?, NULL) """, (en_clean, textbook, unit, safe_context, w_type)) en_id = cursor.lastrowid # 3. Form unique single translation row mapping bond cursor.execute(""" INSERT INTO translations (source_phrase_id, target_phrase_id, deck_name) VALUES (?, ?, 'General') """, (es_id, en_id)) inserted_count += 1 conn.commit() conn.close() print(f"🎉 Schema extraction pipeline completed! Saved {inserted_count} pristine translation pairs.")