# core/clean_glossary.py import re from database.connection import get_connection class GlossaryCleaner: def __init__(self): # Captures trailing markers: " m", " f", " m, pl", " f, pl", " pl" self.gender_pattern = re.compile(r'\s+\b(m|f|m,\s*pl|f,\s*pl|pl)\b\s*$') # Captures verb irregular brackets: " (zc)", " (ie)", etc. self.verb_pattern = re.compile(r'\s*\(([a-z\s]+)\)') def extract_metadata(self, text: str) -> tuple[str, str, str]: """ Parses text to isolate the clean conversational string, the word classification type, and specific grammatical notes. """ word_type = "phrase" grammar_note = None clean_text = text.strip() # Check for verb present-tense irregular markers verb_match = self.verb_pattern.search(clean_text) if verb_match: word_type = "verb" grammar_note = verb_match.group(1) # e.g., 'zc', 'ie' clean_text = self.verb_pattern.sub('', clean_text).strip() return clean_text, word_type, grammar_note # Check for noun gender indicators gender_match = self.gender_pattern.search(clean_text) if gender_match: word_type = "noun" grammar_note = gender_match.group(1).strip() # e.g., 'm', 'f' clean_text = self.gender_pattern.sub('', clean_text).strip() return clean_text, word_type, grammar_note # If it's a single word without tags, check if it's likely an adjective/noun split if ' ' not in clean_text and '/' in clean_text: word_type = "adjective" return clean_text, word_type, grammar_note def ensure_columns_exist(self, cursor): """Dynamically appends schema metadata columns to phrases table if missing.""" try: cursor.execute("ALTER TABLE phrases ADD COLUMN word_type TEXT") cursor.execute("ALTER TABLE phrases ADD COLUMN grammar_note TEXT") except Exception: # Columns already exist, skip safe alert safely pass def process_database_clean(self): conn = get_connection() cursor = conn.cursor() # Ensure our rich metadata slots exist in SQLite self.ensure_columns_exist(cursor) print("๐Ÿงน Extracting raw glossary dataset for metadata preservation...") cursor.execute(""" SELECT t.source_phrase_id, p1.text as es_text, p2.text as en_text, p1.textbook, p1.unit, p1.source_context FROM translations t JOIN phrases p1 ON t.source_phrase_id = p1.id JOIN phrases p2 ON t.target_phrase_id = p2.id WHERE p1.language = 'es' """) raw_rows = cursor.fetchall() if not raw_rows: print("โš ๏ธ No base phrases found to clean.") conn.close() return print(f"โš™๏ธ Migrating {len(raw_rows)} rows into a structured format...") # Clear out current tables to run a fresh, structured reload cursor.execute("DELETE FROM translations") cursor.execute("DELETE FROM phrases") cursor.execute("DELETE FROM audio_tracks") inserted_count = 0 for _, es_raw, en_raw, textbook, unit, context in raw_rows: # Isolate text from technical indicators clean_es, word_type, grammar_note = self.extract_metadata(es_raw) # Handle dual-gender expansions like "apasionado/a" variants = [] if '/' in clean_es: base, suffix = clean_es.split('/', 1) base = base.strip() suffix = suffix.strip() if suffix == 'a' and base.endswith('o'): variants.append((base, "adjective", "m")) variants.append((base[:-1] + 'a', "adjective", "f")) elif suffix == 'ra' and base.endswith('r'): variants.append((base, "adjective", "m")) variants.append((base + 'a', "adjective", "f")) else: variants.append((clean_es, word_type, grammar_note)) else: variants.append((clean_es, word_type, grammar_note)) for es_variant, w_type, g_note in variants: try: # 1. Insert Spanish entry with structural metadata columns filled cursor.execute(""" INSERT INTO phrases (text, language, textbook, unit, source_context, word_type, grammar_note) VALUES (?, 'es', ?, ?, ?, ?, ?) """, (es_variant, textbook, unit, context, w_type, g_note)) es_id = cursor.lastrowid # 2. Insert English entry cursor.execute(""" INSERT INTO phrases (text, language, textbook, unit, source_context, word_type, grammar_note) VALUES (?, 'en', ?, ?, ?, ?, NULL) """, (en_raw, textbook, unit, context, w_type)) en_id = cursor.lastrowid # 3. Create Bidirectional Cross-References cursor.execute("INSERT INTO translations (source_phrase_id, target_phrase_id) VALUES (?, ?)", (es_id, en_id)) cursor.execute("INSERT INTO translations (source_phrase_id, target_phrase_id) VALUES (?, ?)", (en_id, es_id)) inserted_count += 1 except Exception: continue conn.commit() conn.close() print(f"๐ŸŽ‰ Metadata-aware normalization complete! Saved {inserted_count} structured phrases.")