# core/clean_glossary.py import re from database.connection import get_connection class GlossaryCleaner: def __init__(self): # Captures verb irregular brackets: " (zc)", " (ie)", etc. self.verb_pattern = re.compile(r'\s*\(([a-z\s]+)\)') # Aggressive character-level match for broken trailing gender tags # Tracks variations like: " m", " f", "mpl", "fpl", "smpl", "osmpl" at the end of a string self.broken_gender_pattern = re.compile(r'[\s]*\b(m|f|pl)\b$|[\s\w]*(m|f|pl|mpl|fpl|smpl|osmpl)$', re.IGNORECASE) def clean_and_expand_spanish(self, text: str) -> tuple[list[str], str, str]: """Parses and strips layout noise from Spanish strings, extracting rich metadata.""" word_type = "phrase" grammar_note = None clean_text = text.strip() # 1. Extract verb irregularities if present verb_match = self.verb_pattern.search(clean_text) if verb_match: word_type = "verb" grammar_note = verb_match.group(1).strip() clean_text = self.verb_pattern.sub('', clean_text).strip() # 2. Extract and remove sticky gender notations (e.g., "bañ osmpl" -> "baños") # Check if text ends with common markers lower_text = clean_text.lower() if lower_text.endswith(('mpl', 'fpl', 'smpl', 'osmpl')): word_type = "noun" if 'f' in lower_text: grammar_note = "f, pl" else: grammar_note = "m, pl" # Reconstruct the original word base before the layout break # e.g., "bañ osmpl" -> remove "osmpl" and append "os" to restore "baños" if lower_text.endswith('osmpl') and clean_text.lower().endswith('osmpl'): clean_text = clean_text[:-5] + "os" else: # General strip of trailing garbage characters clean_text = re.sub(r'\s*[a-zA-Z\s]*$', '', clean_text) # Standard boundary check for clean tags (e.g., "años60 m") elif re.search(r'\s+\b(m|f|pl|m,\s*pl|f,\s*pl)\b\s*$', clean_text): word_type = "noun" match = re.search(r'\b(m|f|pl|m,\s*pl|f,\s*pl)\b\s*$', clean_text) if match: grammar_note = match.group(1).strip() clean_text = re.sub(r'\s+\b(m|f|pl|m,\s*pl|f,\s*pl)\b\s*$', '', clean_text).strip() clean_text = clean_text.strip() # 3. Expand dual-gender adjectives (e.g., "apasionado/a") variants = [] if '/' in clean_text and ' ' not in clean_text: word_type = "adjective" base, suffix = clean_text.split('/', 1) base = base.strip() suffix = suffix.strip() if suffix == 'a' and base.endswith('o'): variants.append((base, word_type, "m")) variants.append((base[:-1] + 'a', word_type, "f")) elif suffix == 'ra' and base.endswith('r'): variants.append((base, word_type, "m")) variants.append((base + 'a', word_type, "f")) else: variants.append((clean_text, word_type, grammar_note)) else: variants.append((clean_text, word_type, grammar_note)) return variants def clean_english_text(self, text: str) -> str: """Fixes layout spacing issues on English infinitive verbs.""" cleaned = text.strip() # Ensure duplicate internal spacing is compressed cleaned = re.sub(r'\s+', ' ', cleaned) # Intercept smashed English infinitives (e.g., "toappear" -> "to appear") if cleaned.startswith("to") and len(cleaned) > 2 and not cleaned.startswith("to "): cleaned = re.sub(r'^to([a-z])', r'to \1', cleaned) return cleaned def process_database_clean(self): """Processes and normalizes raw table entries into pristine structures.""" conn = get_connection() cursor = conn.cursor() print("🧹 Extracting raw dataset for deep metadata extraction...") cursor.execute(""" SELECT t.source_phrase_id, p1.text as es_text, p2.text as en_text, p1.textbook, p1.unit, p1.source_context FROM translations t JOIN phrases p1 ON t.source_phrase_id = p1.id JOIN phrases p2 ON t.target_phrase_id = p2.id WHERE p1.language = 'es' """) raw_rows = cursor.fetchall() if not raw_rows: print("⚠️ No base phrases found to clean.") conn.close() return print(f"⚙️ Migrating {len(raw_rows)} rows into corrected schemas...") # Clear out previous passes completely cursor.execute("DELETE FROM translations") cursor.execute("DELETE FROM phrases") cursor.execute("DELETE FROM audio_tracks") inserted_count = 0 for _, es_raw, en_raw, textbook, unit, context in raw_rows: # Process Spanish layout text elements es_variants = self.clean_and_expand_spanish(es_raw) # Process and restore spaces to English text elements en_clean = self.clean_english_text(en_raw) for es_clean, w_type, g_note in es_variants: try: # Insert pristine Spanish entry cursor.execute(""" INSERT INTO phrases (text, language, textbook, unit, source_context, word_type, grammar_note) VALUES (?, 'es', ?, ?, ?, ?, ?) """, (es_clean, textbook, unit, context, w_type, g_note)) es_id = cursor.lastrowid # Insert spaced English entry cursor.execute(""" INSERT INTO phrases (text, language, textbook, unit, source_context, word_type, grammar_note) VALUES (?, 'en', ?, ?, ?, ?, NULL) """, (en_clean, textbook, unit, context, w_type)) en_id = cursor.lastrowid # Re-map relations cursor.execute("INSERT INTO translations (source_phrase_id, target_phrase_id) VALUES (?, ?)", (es_id, en_id)) cursor.execute("INSERT INTO translations (source_phrase_id, target_phrase_id) VALUES (?, ?)", (en_id, es_id)) inserted_count += 1 except Exception: continue conn.commit() conn.close() print(f"🎉 Schema extraction pipeline completed! Saved {inserted_count} pristine phrase definitions.")