133 lines
No EOL
5.7 KiB
Python
133 lines
No EOL
5.7 KiB
Python
# core/clean_glossary.py
|
|
import re
|
|
from database.connection import get_connection
|
|
|
|
class GlossaryCleaner:
|
|
def __init__(self):
|
|
# Captures trailing markers: " m", " f", " m, pl", " f, pl", " pl"
|
|
self.gender_pattern = re.compile(r'\s+\b(m|f|m,\s*pl|f,\s*pl|pl)\b\s*$')
|
|
# Captures verb irregular brackets: " (zc)", " (ie)", etc.
|
|
self.verb_pattern = re.compile(r'\s*\(([a-z\s]+)\)')
|
|
|
|
def extract_metadata(self, text: str) -> tuple[str, str, str]:
|
|
"""
|
|
Parses text to isolate the clean conversational string,
|
|
the word classification type, and specific grammatical notes.
|
|
"""
|
|
word_type = "phrase"
|
|
grammar_note = None
|
|
clean_text = text.strip()
|
|
|
|
# Check for verb present-tense irregular markers
|
|
verb_match = self.verb_pattern.search(clean_text)
|
|
if verb_match:
|
|
word_type = "verb"
|
|
grammar_note = verb_match.group(1) # e.g., 'zc', 'ie'
|
|
clean_text = self.verb_pattern.sub('', clean_text).strip()
|
|
return clean_text, word_type, grammar_note
|
|
|
|
# Check for noun gender indicators
|
|
gender_match = self.gender_pattern.search(clean_text)
|
|
if gender_match:
|
|
word_type = "noun"
|
|
grammar_note = gender_match.group(1).strip() # e.g., 'm', 'f'
|
|
clean_text = self.gender_pattern.sub('', clean_text).strip()
|
|
return clean_text, word_type, grammar_note
|
|
|
|
# If it's a single word without tags, check if it's likely an adjective/noun split
|
|
if ' ' not in clean_text and '/' in clean_text:
|
|
word_type = "adjective"
|
|
|
|
return clean_text, word_type, grammar_note
|
|
|
|
def ensure_columns_exist(self, cursor):
|
|
"""Dynamically appends schema metadata columns to phrases table if missing."""
|
|
try:
|
|
cursor.execute("ALTER TABLE phrases ADD COLUMN word_type TEXT")
|
|
cursor.execute("ALTER TABLE phrases ADD COLUMN grammar_note TEXT")
|
|
except Exception:
|
|
# Columns already exist, skip safe alert safely
|
|
pass
|
|
|
|
def process_database_clean(self):
|
|
conn = get_connection()
|
|
cursor = conn.cursor()
|
|
|
|
# Ensure our rich metadata slots exist in SQLite
|
|
self.ensure_columns_exist(cursor)
|
|
|
|
print("🧹 Extracting raw glossary dataset for metadata preservation...")
|
|
|
|
cursor.execute("""
|
|
SELECT t.source_phrase_id, p1.text as es_text, p2.text as en_text,
|
|
p1.textbook, p1.unit, p1.source_context
|
|
FROM translations t
|
|
JOIN phrases p1 ON t.source_phrase_id = p1.id
|
|
JOIN phrases p2 ON t.target_phrase_id = p2.id
|
|
WHERE p1.language = 'es'
|
|
""")
|
|
raw_rows = cursor.fetchall()
|
|
|
|
if not raw_rows:
|
|
print("⚠️ No base phrases found to clean.")
|
|
conn.close()
|
|
return
|
|
|
|
print(f"⚙️ Migrating {len(raw_rows)} rows into a structured format...")
|
|
|
|
# Clear out current tables to run a fresh, structured reload
|
|
cursor.execute("DELETE FROM translations")
|
|
cursor.execute("DELETE FROM phrases")
|
|
cursor.execute("DELETE FROM audio_tracks")
|
|
|
|
inserted_count = 0
|
|
|
|
for _, es_raw, en_raw, textbook, unit, context in raw_rows:
|
|
# Isolate text from technical indicators
|
|
clean_es, word_type, grammar_note = self.extract_metadata(es_raw)
|
|
|
|
# Handle dual-gender expansions like "apasionado/a"
|
|
variants = []
|
|
if '/' in clean_es:
|
|
base, suffix = clean_es.split('/', 1)
|
|
base = base.strip()
|
|
suffix = suffix.strip()
|
|
|
|
if suffix == 'a' and base.endswith('o'):
|
|
variants.append((base, "adjective", "m"))
|
|
variants.append((base[:-1] + 'a', "adjective", "f"))
|
|
elif suffix == 'ra' and base.endswith('r'):
|
|
variants.append((base, "adjective", "m"))
|
|
variants.append((base + 'a', "adjective", "f"))
|
|
else:
|
|
variants.append((clean_es, word_type, grammar_note))
|
|
else:
|
|
variants.append((clean_es, word_type, grammar_note))
|
|
|
|
for es_variant, w_type, g_note in variants:
|
|
try:
|
|
# 1. Insert Spanish entry with structural metadata columns filled
|
|
cursor.execute("""
|
|
INSERT INTO phrases (text, language, textbook, unit, source_context, word_type, grammar_note)
|
|
VALUES (?, 'es', ?, ?, ?, ?, ?)
|
|
""", (es_variant, textbook, unit, context, w_type, g_note))
|
|
es_id = cursor.lastrowid
|
|
|
|
# 2. Insert English entry
|
|
cursor.execute("""
|
|
INSERT INTO phrases (text, language, textbook, unit, source_context, word_type, grammar_note)
|
|
VALUES (?, 'en', ?, ?, ?, ?, NULL)
|
|
""", (en_raw, textbook, unit, context, w_type))
|
|
en_id = cursor.lastrowid
|
|
|
|
# 3. Create Bidirectional Cross-References
|
|
cursor.execute("INSERT INTO translations (source_phrase_id, target_phrase_id) VALUES (?, ?)", (es_id, en_id))
|
|
cursor.execute("INSERT INTO translations (source_phrase_id, target_phrase_id) VALUES (?, ?)", (en_id, es_id))
|
|
|
|
inserted_count += 1
|
|
except Exception:
|
|
continue
|
|
|
|
conn.commit()
|
|
conn.close()
|
|
print(f"🎉 Metadata-aware normalization complete! Saved {inserted_count} structured phrases.") |