diff --git a/core/clean_glossary.py b/core/clean_glossary.py index 4af4aae..71d86f4 100644 --- a/core/clean_glossary.py +++ b/core/clean_glossary.py @@ -4,59 +4,93 @@ from database.connection import get_connection class GlossaryCleaner: def __init__(self): - # Captures trailing markers: " m", " f", " m, pl", " f, pl", " pl" - self.gender_pattern = re.compile(r'\s+\b(m|f|m,\s*pl|f,\s*pl|pl)\b\s*$') # Captures verb irregular brackets: " (zc)", " (ie)", etc. self.verb_pattern = re.compile(r'\s*\(([a-z\s]+)\)') + + # Aggressive character-level match for broken trailing gender tags + # Tracks variations like: " m", " f", "mpl", "fpl", "smpl", "osmpl" at the end of a string + self.broken_gender_pattern = re.compile(r'[\s]*\b(m|f|pl)\b$|[\s\w]*(m|f|pl|mpl|fpl|smpl|osmpl)$', re.IGNORECASE) - def extract_metadata(self, text: str) -> tuple[str, str, str]: - """ - Parses text to isolate the clean conversational string, - the word classification type, and specific grammatical notes. - """ + def clean_and_expand_spanish(self, text: str) -> tuple[list[str], str, str]: + """Parses and strips layout noise from Spanish strings, extracting rich metadata.""" word_type = "phrase" grammar_note = None clean_text = text.strip() - # Check for verb present-tense irregular markers + # 1. Extract verb irregularities if present verb_match = self.verb_pattern.search(clean_text) if verb_match: word_type = "verb" - grammar_note = verb_match.group(1) # e.g., 'zc', 'ie' + grammar_note = verb_match.group(1).strip() clean_text = self.verb_pattern.sub('', clean_text).strip() - return clean_text, word_type, grammar_note - # Check for noun gender indicators - gender_match = self.gender_pattern.search(clean_text) - if gender_match: + # 2. Extract and remove sticky gender notations (e.g., "bañ osmpl" -> "baños") + # Check if text ends with common markers + lower_text = clean_text.lower() + if lower_text.endswith(('mpl', 'fpl', 'smpl', 'osmpl')): word_type = "noun" - grammar_note = gender_match.group(1).strip() # e.g., 'm', 'f' - clean_text = self.gender_pattern.sub('', clean_text).strip() - return clean_text, word_type, grammar_note - - # If it's a single word without tags, check if it's likely an adjective/noun split - if ' ' not in clean_text and '/' in clean_text: - word_type = "adjective" + if 'f' in lower_text: + grammar_note = "f, pl" + else: + grammar_note = "m, pl" - return clean_text, word_type, grammar_note + # Reconstruct the original word base before the layout break + # e.g., "bañ osmpl" -> remove "osmpl" and append "os" to restore "baños" + if lower_text.endswith('osmpl') and clean_text.lower().endswith('osmpl'): + clean_text = clean_text[:-5] + "os" + else: + # General strip of trailing garbage characters + clean_text = re.sub(r'\s*[a-zA-Z\s]*$', '', clean_text) + + # Standard boundary check for clean tags (e.g., "años60 m") + elif re.search(r'\s+\b(m|f|pl|m,\s*pl|f,\s*pl)\b\s*$', clean_text): + word_type = "noun" + match = re.search(r'\b(m|f|pl|m,\s*pl|f,\s*pl)\b\s*$', clean_text) + if match: + grammar_note = match.group(1).strip() + clean_text = re.sub(r'\s+\b(m|f|pl|m,\s*pl|f,\s*pl)\b\s*$', '', clean_text).strip() - def ensure_columns_exist(self, cursor): - """Dynamically appends schema metadata columns to phrases table if missing.""" - try: - cursor.execute("ALTER TABLE phrases ADD COLUMN word_type TEXT") - cursor.execute("ALTER TABLE phrases ADD COLUMN grammar_note TEXT") - except Exception: - # Columns already exist, skip safe alert safely - pass + clean_text = clean_text.strip() + + # 3. Expand dual-gender adjectives (e.g., "apasionado/a") + variants = [] + if '/' in clean_text and ' ' not in clean_text: + word_type = "adjective" + base, suffix = clean_text.split('/', 1) + base = base.strip() + suffix = suffix.strip() + + if suffix == 'a' and base.endswith('o'): + variants.append((base, word_type, "m")) + variants.append((base[:-1] + 'a', word_type, "f")) + elif suffix == 'ra' and base.endswith('r'): + variants.append((base, word_type, "m")) + variants.append((base + 'a', word_type, "f")) + else: + variants.append((clean_text, word_type, grammar_note)) + else: + variants.append((clean_text, word_type, grammar_note)) + + return variants + + def clean_english_text(self, text: str) -> str: + """Fixes layout spacing issues on English infinitive verbs.""" + cleaned = text.strip() + # Ensure duplicate internal spacing is compressed + cleaned = re.sub(r'\s+', ' ', cleaned) + + # Intercept smashed English infinitives (e.g., "toappear" -> "to appear") + if cleaned.startswith("to") and len(cleaned) > 2 and not cleaned.startswith("to "): + cleaned = re.sub(r'^to([a-z])', r'to \1', cleaned) + + return cleaned def process_database_clean(self): + """Processes and normalizes raw table entries into pristine structures.""" conn = get_connection() cursor = conn.cursor() - # Ensure our rich metadata slots exist in SQLite - self.ensure_columns_exist(cursor) - - print("🧹 Extracting raw glossary dataset for metadata preservation...") + print("🧹 Extracting raw dataset for deep metadata extraction...") cursor.execute(""" SELECT t.source_phrase_id, p1.text as es_text, p2.text as en_text, @@ -73,9 +107,9 @@ class GlossaryCleaner: conn.close() return - print(f"⚙️ Migrating {len(raw_rows)} rows into a structured format...") + print(f"⚙️ Migrating {len(raw_rows)} rows into corrected schemas...") - # Clear out current tables to run a fresh, structured reload + # Clear out previous passes completely cursor.execute("DELETE FROM translations") cursor.execute("DELETE FROM phrases") cursor.execute("DELETE FROM audio_tracks") @@ -83,44 +117,28 @@ class GlossaryCleaner: inserted_count = 0 for _, es_raw, en_raw, textbook, unit, context in raw_rows: - # Isolate text from technical indicators - clean_es, word_type, grammar_note = self.extract_metadata(es_raw) + # Process Spanish layout text elements + es_variants = self.clean_and_expand_spanish(es_raw) + # Process and restore spaces to English text elements + en_clean = self.clean_english_text(en_raw) - # Handle dual-gender expansions like "apasionado/a" - variants = [] - if '/' in clean_es: - base, suffix = clean_es.split('/', 1) - base = base.strip() - suffix = suffix.strip() - - if suffix == 'a' and base.endswith('o'): - variants.append((base, "adjective", "m")) - variants.append((base[:-1] + 'a', "adjective", "f")) - elif suffix == 'ra' and base.endswith('r'): - variants.append((base, "adjective", "m")) - variants.append((base + 'a', "adjective", "f")) - else: - variants.append((clean_es, word_type, grammar_note)) - else: - variants.append((clean_es, word_type, grammar_note)) - - for es_variant, w_type, g_note in variants: + for es_clean, w_type, g_note in es_variants: try: - # 1. Insert Spanish entry with structural metadata columns filled + # Insert pristine Spanish entry cursor.execute(""" INSERT INTO phrases (text, language, textbook, unit, source_context, word_type, grammar_note) VALUES (?, 'es', ?, ?, ?, ?, ?) - """, (es_variant, textbook, unit, context, w_type, g_note)) + """, (es_clean, textbook, unit, context, w_type, g_note)) es_id = cursor.lastrowid - # 2. Insert English entry + # Insert spaced English entry cursor.execute(""" INSERT INTO phrases (text, language, textbook, unit, source_context, word_type, grammar_note) VALUES (?, 'en', ?, ?, ?, ?, NULL) - """, (en_raw, textbook, unit, context, w_type)) + """, (en_clean, textbook, unit, context, w_type)) en_id = cursor.lastrowid - # 3. Create Bidirectional Cross-References + # Re-map relations cursor.execute("INSERT INTO translations (source_phrase_id, target_phrase_id) VALUES (?, ?)", (es_id, en_id)) cursor.execute("INSERT INTO translations (source_phrase_id, target_phrase_id) VALUES (?, ?)", (en_id, es_id)) @@ -130,4 +148,4 @@ class GlossaryCleaner: conn.commit() conn.close() - print(f"🎉 Metadata-aware normalization complete! Saved {inserted_count} structured phrases.") \ No newline at end of file + print(f"🎉 Schema extraction pipeline completed! Saved {inserted_count} pristine phrase definitions.") \ No newline at end of file diff --git a/database/connection.py b/database/connection.py index de6b28c..e1dbe0a 100644 --- a/database/connection.py +++ b/database/connection.py @@ -1,59 +1,54 @@ # database/connection.py import sqlite3 -import os - -DB_NAME = "spanish_trainer.db" def get_connection(): - """Returns a standard connection object to the SQLite database.""" - return sqlite3.connect(DB_NAME) + return sqlite3.connect("spanish_trainer.db") def init_db(): - """ - Initializes the SQLite database tables if they do not exist. - This safely runs on every boot without wiping your existing data. - """ - print(f"🗄️ Checking database status for '{DB_NAME}'...") - - # The SQL schema we designed for your glossary, cross-references, and tracks - schema = """ - CREATE TABLE IF NOT EXISTS phrases ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - text TEXT NOT NULL, - language TEXT NOT NULL, - textbook TEXT DEFAULT NULL, - unit INTEGER DEFAULT NULL, - source_context TEXT DEFAULT NULL, - created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP - ); - - CREATE TABLE IF NOT EXISTS translations ( - source_phrase_id INTEGER, - target_phrase_id INTEGER, - PRIMARY KEY (source_phrase_id, target_phrase_id), - FOREIGN KEY (source_phrase_id) REFERENCES phrases(id) ON DELETE CASCADE, - FOREIGN KEY (target_phrase_id) REFERENCES phrases(id) ON DELETE CASCADE - ); - - CREATE TABLE IF NOT EXISTS audio_tracks ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - phrase_id INTEGER NOT NULL, - voice_gender TEXT NOT NULL, - voice_name TEXT NOT NULL, - file_path TEXT NOT NULL, - is_reference INTEGER DEFAULT 1, - FOREIGN KEY (phrase_id) REFERENCES phrases(id) ON DELETE CASCADE - ); - """ - conn = get_connection() - try: - cursor = conn.cursor() - # executescript allows running multiple CREATE TABLE statements at once - cursor.executescript(schema) - conn.commit() - print("✅ Database tables verified and initialized successfully.") - except sqlite3.Error as e: - print(f"❌ Database initialization failed: {e}") - finally: - conn.close() \ No newline at end of file + cursor = conn.cursor() + + # 1. Main Phrases Table + cursor.execute(""" + CREATE TABLE IF NOT EXISTS phrases ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + text TEXT NOT NULL, + language TEXT NOT NULL, + textbook TEXT, + unit INTEGER, + source_context TEXT, + word_type TEXT, + grammar_note TEXT, + voice_gender TEXT DEFAULT 'female', + base_speed REAL DEFAULT 1.0, + deck_name TEXT DEFAULT 'General', + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP + ) + """) + + # 2. Translations Cross-Reference Table + cursor.execute(""" + CREATE TABLE IF NOT EXISTS translations ( + source_phrase_id INTEGER, + target_phrase_id INTEGER, + PRIMARY KEY (source_phrase_id, target_phrase_id), + FOREIGN KEY (source_phrase_id) REFERENCES phrases(id) ON DELETE CASCADE, + FOREIGN KEY (target_phrase_id) REFERENCES phrases(id) ON DELETE CASCADE + ) + """) + + # 3. Audio Tracks Metadata Table (Required by the glossary cleaner) + cursor.execute(""" + CREATE TABLE IF NOT EXISTS audio_tracks ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + phrase_id INTEGER, + file_path TEXT NOT NULL, + sample_rate INTEGER, + duration REAL, + FOREIGN KEY (phrase_id) REFERENCES phrases(id) ON DELETE CASCADE + ) + """) + + conn.commit() + conn.close() + print("✅ Rich metadata database schema initialized successfully.") diff --git a/main.py b/main.py index 79051e5..69253c8 100644 --- a/main.py +++ b/main.py @@ -1,27 +1,410 @@ # main.py -import asyncio -from database.connection import init_db +import sys +import os +from PyQt6.QtWidgets import ( + QApplication, QMainWindow, QWidget, QTabWidget, QVBoxLayout, + QHBoxLayout, QLabel, QPushButton, QLineEdit, QComboBox, + QTableWidget, QTableWidgetItem, QSlider, QFormLayout, QTextEdit, QFrame, QMessageBox +) +from PyQt6.QtCore import Qt, QUrl +from PyQt6.QtMultimedia import QMediaPlayer, QAudioOutput +from PyQt6.QtGui import QFont +from database.connection import init_db, get_connection from core.bulk_importer import BulkImporter from core.clean_glossary import GlossaryCleaner -async def rebuild_pipeline(): - print("🚀 Initiating Clean Reconstruction Pipeline...") +class SpanishTrainerApp(QMainWindow): + def __init__(self): + super().__init__() + self.setWindowTitle("Castilian Voice Trainer Pro") + self.setMinimumSize(1000, 650) + + # 1. Initialize and Seed Database if Empty + self.ensure_database_populated() + + # 2. Initialize PyQt Multimedia Audio Engine Components + self.media_player = QMediaPlayer() + self.audio_output = QAudioOutput() + self.media_player.setAudioOutput(self.audio_output) + + self.current_flashcard_id = None + + # 3. Build UI Components + self.tabs = QTabWidget() + self.setCentralWidget(self.tabs) + + self.init_phrase_sandbox_tab() + self.init_flashcard_reviewer_tab() + + # 4. Initial Load of Database Data into Grid View + self.refresh_crud_table() + + def ensure_database_populated(self): + """Verifies if the database exists and has records; seeds it from the PDF if empty.""" + init_db() + + conn = get_connection() + cursor = conn.cursor() + try: + cursor.execute("SELECT COUNT(*) FROM phrases") + count = cursor.fetchone()[0] + except Exception: + count = 0 + conn.close() + + if count == 0: + print("🗄️ Database appears empty. Running the structural reconstruction pipeline...") + pdf_file = "aula_int_plus_1_glos_en_alfa.pdf" + textbook = "Aula Internacional Plus 1" + + if os.path.exists(pdf_file): + importer = BulkImporter() + importer.import_pdf_glossary(pdf_file, textbook) + + cleaner = GlossaryCleaner() + cleaner.process_database_clean() + print("✨ Database successfully seeded with metadata-parsed entries.") + else: + print(f"⚠️ Warning: Could not find '{pdf_file}' to automatically seed records.") + + # ===================================================================== + # 🗄️ TAB 1: PHRASE SANDBOX (CRUD Panel) + # ===================================================================== + def init_phrase_sandbox_tab(self): + tab = QWidget() + layout = QHBoxLayout(tab) + + left_panel = QVBoxLayout() + search_layout = QHBoxLayout() + search_layout.addWidget(QLabel("🔍 Filter text:")) + self.search_input = QLineEdit() + self.search_input.textChanged.connect(self.refresh_crud_table) + search_layout.addWidget(self.search_input) + left_panel.addLayout(search_layout) + + self.phrase_table = QTableWidget() + self.phrase_table.setColumnCount(6) + self.phrase_table.setHorizontalHeaderLabels(["ID", "Text", "Lang", "Type", "Voice Gender", "Deck Tag"]) + self.phrase_table.itemSelectionChanged.connect(self.handle_table_row_select) + left_panel.addWidget(self.phrase_table) + + right_panel = QVBoxLayout() + form_frame = QFrame() + form_frame.setFrameShape(QFrame.Shape.StyledPanel) + form_layout = QFormLayout(form_frame) + + self.input_id = QLineEdit() + self.input_id.setReadOnly(True) + self.input_id.setPlaceholderText("Auto-assigned ID") + + self.input_text = QTextEdit() + self.input_text.setMaximumHeight(60) + + self.combo_lang = QComboBox() + self.combo_lang.addItems(["es", "en"]) + + self.combo_type = QComboBox() + self.combo_type.addItems(["sentence", "phrase", "noun", "verb", "adjective"]) + + self.combo_voice_gender = QComboBox() + self.combo_voice_gender.addItems(["female", "male"]) + + self.slider_base_speed = QSlider(Qt.Orientation.Horizontal) + self.slider_base_speed.setMinimum(50) + self.slider_base_speed.setMaximum(150) + self.slider_base_speed.setValue(100) + self.lbl_base_speed = QLabel("1.00x") + self.slider_base_speed.valueChanged.connect(lambda v: self.lbl_base_speed.setText(f"{v/100:.2f}x")) + + speed_box = QHBoxLayout() + speed_box.addWidget(self.slider_base_speed) + speed_box.addWidget(self.lbl_base_speed) + + self.input_deck_tag = QLineEdit() + self.input_deck_tag.setPlaceholderText("e.g., Anki_Unit_1") + + form_layout.addRow("Phrase ID Resource:", self.input_id) + form_layout.addRow(QLabel("Target Conversational Text:")) + form_layout.addRow(self.input_text) + form_layout.addRow("Language Accent:", self.combo_lang) + form_layout.addRow("Grammar Type Classification:", self.combo_type) + form_layout.addRow("Preferred Voice Gender:", self.combo_voice_gender) + form_layout.addRow("Default Playback Speed:", speed_box) + form_layout.addRow("Deck / Group Identifier Tag:", self.input_deck_tag) + + crud_buttons = QHBoxLayout() + self.btn_save = QPushButton("➕ Save New") + self.btn_update = QPushButton("💾 Update Entry") + self.btn_delete = QPushButton("🗑️ Delete") + + self.btn_save.clicked.connect(self.crud_create) + self.btn_update.clicked.connect(self.crud_update) + self.btn_delete.clicked.connect(self.crud_delete) + + crud_buttons.addWidget(self.btn_save) + crud_buttons.addWidget(self.btn_update) + crud_buttons.addWidget(self.btn_delete) + + right_panel.addWidget(QLabel("