139_spanish-voice-trainer/core/clean_glossary.py

# core/clean_glossary.py
import re
from database.connection import get_connection

class GlossaryCleaner:
    def __init__(self):
        # Captures verb irregular brackets: " (zc)", " (ie)", etc.
        self.verb_pattern = re.compile(r'\s*\(([a-z\s]+)\)')
        
        # Aggressive character-level match for broken trailing gender tags
        # Tracks variations like: " m", " f", "mpl", "fpl", "smpl", "osmpl" at the end of a string
        self.broken_gender_pattern = re.compile(r'[\s]*\b(m|f|pl)\b$|[\s\w]*(m|f|pl|mpl|fpl|smpl|osmpl)$', re.IGNORECASE)

    def clean_and_expand_spanish(self, text: str) -> tuple[list[str], str, str]:
        """Parses and strips layout noise from Spanish strings, extracting rich metadata."""
        word_type = "phrase"
        grammar_note = None
        clean_text = text.strip()

        # 1. Extract verb irregularities if present
        verb_match = self.verb_pattern.search(clean_text)
        if verb_match:
            word_type = "verb"
            grammar_note = verb_match.group(1).strip()
            clean_text = self.verb_pattern.sub('', clean_text).strip()

        # 2. Extract and remove sticky gender notations (e.g., "bañ osmpl" -> "baños")
        # Check if text ends with common markers
        lower_text = clean_text.lower()
        if lower_text.endswith(('mpl', 'fpl', 'smpl', 'osmpl')):
            word_type = "noun"
            if 'f' in lower_text:
                grammar_note = "f, pl"
            else:
                grammar_note = "m, pl"
            
            # Reconstruct the original word base before the layout break
            # e.g., "bañ osmpl" -> remove "osmpl" and append "os" to restore "baños"
            if lower_text.endswith('osmpl') and clean_text.lower().endswith('osmpl'):
                clean_text = clean_text[:-5] + "os"
            else:
                # General strip of trailing garbage characters
                clean_text = re.sub(r'\s*[a-zA-Z\s]*$', '', clean_text)
        
        # Standard boundary check for clean tags (e.g., "años60 m")
        elif re.search(r'\s+\b(m|f|pl|m,\s*pl|f,\s*pl)\b\s*$', clean_text):
            word_type = "noun"
            match = re.search(r'\b(m|f|pl|m,\s*pl|f,\s*pl)\b\s*$', clean_text)
            if match:
                grammar_note = match.group(1).strip()
            clean_text = re.sub(r'\s+\b(m|f|pl|m,\s*pl|f,\s*pl)\b\s*$', '', clean_text).strip()

        clean_text = clean_text.strip()

        # 3. Expand dual-gender adjectives (e.g., "apasionado/a")
        variants = []
        if '/' in clean_text and ' ' not in clean_text:
            word_type = "adjective"
            base, suffix = clean_text.split('/', 1)
            base = base.strip()
            suffix = suffix.strip()
            
            if suffix == 'a' and base.endswith('o'):
                variants.append((base, word_type, "m"))
                variants.append((base[:-1] + 'a', word_type, "f"))
            elif suffix == 'ra' and base.endswith('r'):
                variants.append((base, word_type, "m"))
                variants.append((base + 'a', word_type, "f"))
            else:
                variants.append((clean_text, word_type, grammar_note))
        else:
            variants.append((clean_text, word_type, grammar_note))

        return variants

    def clean_english_text(self, text: str) -> str:
        """Fixes layout spacing issues on English infinitive verbs."""
        cleaned = text.strip()
        # Ensure duplicate internal spacing is compressed
        cleaned = re.sub(r'\s+', ' ', cleaned)
        
        # Intercept smashed English infinitives (e.g., "toappear" -> "to appear")
        if cleaned.startswith("to") and len(cleaned) > 2 and not cleaned.startswith("to "):
            cleaned = re.sub(r'^to([a-z])', r'to \1', cleaned)
            
        return cleaned

    def process_database_clean(self):
        """Processes and normalizes raw table entries into pristine structures."""
        conn = get_connection()
        cursor = conn.cursor()
        
        print("🧹 Extracting raw dataset for deep metadata extraction...")
        
        cursor.execute("""
            SELECT t.source_phrase_id, p1.text as es_text, p2.text as en_text, 
                   p1.textbook, p1.unit, p1.source_context
            FROM translations t
            JOIN phrases p1 ON t.source_phrase_id = p1.id
            JOIN phrases p2 ON t.target_phrase_id = p2.id
            WHERE p1.language = 'es'
        """)
        raw_rows = cursor.fetchall()
        
        if not raw_rows:
            print("⚠️ No base phrases found to clean.")
            conn.close()
            return

        print(f"⚙️ Migrating {len(raw_rows)} rows into corrected schemas...")
        
        # Clear out previous passes completely
        cursor.execute("DELETE FROM translations")
        cursor.execute("DELETE FROM phrases")
        cursor.execute("DELETE FROM audio_tracks")
        
        inserted_count = 0
        
        for _, es_raw, en_raw, textbook, unit, context in raw_rows:
            # Process Spanish layout text elements
            es_variants = self.clean_and_expand_spanish(es_raw)
            # Process and restore spaces to English text elements
            en_clean = self.clean_english_text(en_raw)
            
            for es_clean, w_type, g_note in es_variants:
                try:
                    # Insert pristine Spanish entry
                    cursor.execute("""
                        INSERT INTO phrases (text, language, textbook, unit, source_context, word_type, grammar_note)
                        VALUES (?, 'es', ?, ?, ?, ?, ?)
                    """, (es_clean, textbook, unit, context, w_type, g_note))
                    es_id = cursor.lastrowid

                    # Insert spaced English entry
                    cursor.execute("""
                        INSERT INTO phrases (text, language, textbook, unit, source_context, word_type, grammar_note)
                        VALUES (?, 'en', ?, ?, ?, ?, NULL)
                    """, (en_clean, textbook, unit, context, w_type))
                    en_id = cursor.lastrowid

                    # Re-map relations
                    cursor.execute("INSERT INTO translations (source_phrase_id, target_phrase_id) VALUES (?, ?)", (es_id, en_id))
                    cursor.execute("INSERT INTO translations (source_phrase_id, target_phrase_id) VALUES (?, ?)", (en_id, es_id))
                    
                    inserted_count += 1
                except Exception:
                    continue

        conn.commit()
        conn.close()
        print(f"🎉 Schema extraction pipeline completed! Saved {inserted_count} pristine phrase definitions.")