added clean_ glossary and more notes

core/clean_glossary.py

@@ -0,0 +1,133 @@
+# core/clean_glossary.py
+import re
+from database.connection import get_connection
+
+class GlossaryCleaner:
+    def __init__(self):
+        # Captures trailing markers: " m", " f", " m, pl", " f, pl", " pl"
+        self.gender_pattern = re.compile(r'\s+\b(m|f|m,\s*pl|f,\s*pl|pl)\b\s*$')
+        # Captures verb irregular brackets: " (zc)", " (ie)", etc.
+        self.verb_pattern = re.compile(r'\s*\(([a-z\s]+)\)')
+
+    def extract_metadata(self, text: str) -> tuple[str, str, str]:
+        """
+        Parses text to isolate the clean conversational string, 
+        the word classification type, and specific grammatical notes.
+        """
+        word_type = "phrase"
+        grammar_note = None
+        clean_text = text.strip()
+
+        # Check for verb present-tense irregular markers
+        verb_match = self.verb_pattern.search(clean_text)
+        if verb_match:
+            word_type = "verb"
+            grammar_note = verb_match.group(1) # e.g., 'zc', 'ie'
+            clean_text = self.verb_pattern.sub('', clean_text).strip()
+            return clean_text, word_type, grammar_note
+
+        # Check for noun gender indicators
+        gender_match = self.gender_pattern.search(clean_text)
+        if gender_match:
+            word_type = "noun"
+            grammar_note = gender_match.group(1).strip() # e.g., 'm', 'f'
+            clean_text = self.gender_pattern.sub('', clean_text).strip()
+            return clean_text, word_type, grammar_note
+
+        # If it's a single word without tags, check if it's likely an adjective/noun split
+        if ' ' not in clean_text and '/' in clean_text:
+            word_type = "adjective"
+            
+        return clean_text, word_type, grammar_note
+
+    def ensure_columns_exist(self, cursor):
+        """Dynamically appends schema metadata columns to phrases table if missing."""
+        try:
+            cursor.execute("ALTER TABLE phrases ADD COLUMN word_type TEXT")
+            cursor.execute("ALTER TABLE phrases ADD COLUMN grammar_note TEXT")
+        except Exception:
+            # Columns already exist, skip safe alert safely
+            pass
+
+    def process_database_clean(self):
+        conn = get_connection()
+        cursor = conn.cursor()
+        
+        # Ensure our rich metadata slots exist in SQLite
+        self.ensure_columns_exist(cursor)
+        
+        print("🧹 Extracting raw glossary dataset for metadata preservation...")
+        
+        cursor.execute("""
+            SELECT t.source_phrase_id, p1.text as es_text, p2.text as en_text, 
+                   p1.textbook, p1.unit, p1.source_context
+            FROM translations t
+            JOIN phrases p1 ON t.source_phrase_id = p1.id
+            JOIN phrases p2 ON t.target_phrase_id = p2.id
+            WHERE p1.language = 'es'
+        """)
+        raw_rows = cursor.fetchall()
+        
+        if not raw_rows:
+            print("⚠️ No base phrases found to clean.")
+            conn.close()
+            return
+
+        print(f"⚙️ Migrating {len(raw_rows)} rows into a structured format...")
+        
+        # Clear out current tables to run a fresh, structured reload
+        cursor.execute("DELETE FROM translations")
+        cursor.execute("DELETE FROM phrases")
+        cursor.execute("DELETE FROM audio_tracks")
+        
+        inserted_count = 0
+        
+        for _, es_raw, en_raw, textbook, unit, context in raw_rows:
+            # Isolate text from technical indicators
+            clean_es, word_type, grammar_note = self.extract_metadata(es_raw)
+            
+            # Handle dual-gender expansions like "apasionado/a"
+            variants = []
+            if '/' in clean_es:
+                base, suffix = clean_es.split('/', 1)
+                base = base.strip()
+                suffix = suffix.strip()
+                
+                if suffix == 'a' and base.endswith('o'):
+                    variants.append((base, "adjective", "m"))
+                    variants.append((base[:-1] + 'a', "adjective", "f"))
+                elif suffix == 'ra' and base.endswith('r'):
+                    variants.append((base, "adjective", "m"))
+                    variants.append((base + 'a', "adjective", "f"))
+                else:
+                    variants.append((clean_es, word_type, grammar_note))
+            else:
+                variants.append((clean_es, word_type, grammar_note))
+
+            for es_variant, w_type, g_note in variants:
+                try:
+                    # 1. Insert Spanish entry with structural metadata columns filled
+                    cursor.execute("""
+                        INSERT INTO phrases (text, language, textbook, unit, source_context, word_type, grammar_note)
+                        VALUES (?, 'es', ?, ?, ?, ?, ?)
+                    """, (es_variant, textbook, unit, context, w_type, g_note))
+                    es_id = cursor.lastrowid
+
+                    # 2. Insert English entry
+                    cursor.execute("""
+                        INSERT INTO phrases (text, language, textbook, unit, source_context, word_type, grammar_note)
+                        VALUES (?, 'en', ?, ?, ?, ?, NULL)
+                    """, (en_raw, textbook, unit, context, w_type))
+                    en_id = cursor.lastrowid
+
+                    # 3. Create Bidirectional Cross-References
+                    cursor.execute("INSERT INTO translations (source_phrase_id, target_phrase_id) VALUES (?, ?)", (es_id, en_id))
+                    cursor.execute("INSERT INTO translations (source_phrase_id, target_phrase_id) VALUES (?, ?)", (en_id, es_id))
+                    
+                    inserted_count += 1
+                except Exception:
+                    continue
+
+        conn.commit()
+        conn.close()
+        print(f"🎉 Metadata-aware normalization complete! Saved {inserted_count} structured phrases.")

doc/Notes.md

@@ -30,6 +30,9 @@
 - [Steps of app creation](#steps-of-app-creation)
 - [Gemini Reposne](#gemini-reposne)
 - [So we fix the bulk upload file](#so-we-fix-the-bulk-upload-file)
+- [Fix up of data](#fix-up-of-data)
+- [Gemini initial response](#gemini-initial-response)
+- [Gemini Response](#gemini-response)
 
 # 1. spanish-voice-trainer
 # 2. Project Summary: 
@@ -411,3 +414,39 @@ v5.0 docling gave us
 | Col 1 (ES) | Col 2 (EN) | Col 3 (Unit) | Col 4 (ES) | Col 5 (EN) | Col 6 (Unit) |
 ```
 # So we fix the bulk upload file 
+```zsh
+git tag -a v6.0 -m "working version of  bulk importer, but it leaves gramar info"
+```
+# Fix up of data
+
+We should probably remove the grammar categorisation which is done.
+5 años60 m es Aula Internacional Plus 1 87 U87A 2026-06-13 03:00:53
+The m
+9 aparecer (zc) es Aula Internacional Plus 1 36 U36A 2026-06-13 03:00:53
+The (zc)
+13 apasionado/a es Aula Internacional Plus 1 515 U515C 2026-06-13 03:00:53
+The /a maybe has be duplicated into two words 
+
+# Gemini initial response
+
+Was was to make a module to clean up the data, I said that we should be keeping the data
+
+# Gemini Response
+
+You raise an exceptional database design point. You are absolutely right: knowing whether a word is a masculine noun, a feminine noun, or an irregular verb is vital linguistic data that we shouldn't just throw away.
+
+The issue isn't that the information is bad, but rather that we were storing it directly inside the text field itself. Keeping it there causes two major problems:
+
+Audio Synthesis Ruin: As mentioned, edge-tts will literally read aloud "años sesenta eme" or "aparecer zeta ce".
+
+Search and Match Failure: If you query your database for the word "aparecer", a direct lookup will fail because the database thinks the word is literally "aparecer (zc)".
+
+The Relational Solution: Keep the Info, Just Move the Slot
+The cleanest engineering approach is to strip the grammar tags out of the conversational text field, but save them into dedicated metadata columns on the exact same row.
+
+Let's modify your SQLite phrases schema to add columns for word_type (noun, verb, adjective, phrase) and grammar_note (m, f, zc, ie). This keeps your data rich and informative, keeps your text search crisp, and lets us pass clean strings to the voice engine.
+
+Here is the plan to gracefully adjust your schema and run an updated, data-preserving migration:
+
+This requires adding another column to our database
+

main.py

@@ -2,19 +2,26 @@
 import asyncio
 from database.connection import init_db
 from core.bulk_importer import BulkImporter
+from core.clean_glossary import GlossaryCleaner
 
-async def main():
+async def rebuild_pipeline():
-    # Verify tables exist or run schemas
+    print("🚀 Initiating Clean Reconstruction Pipeline...")
+    
+    # 1. This will automatically recreate the blank .db file and all tables
     init_db()
     
-    # Initialize the importer pipeline
+    # 2. Run the layout-aware bulk import from the PDF layout text
     importer = BulkImporter()
-    
-    # Execute structural parse and database write operations
     pdf_file = "aula_int_plus_1_glos_en_alfa.pdf"
     textbook = "Aula Internacional Plus 1"
     
     importer.import_pdf_glossary(pdf_file, textbook)
     
+    # 3. Immediately run the metadata preservation and text cleaning pass
+    cleaner = GlossaryCleaner()
+    cleaner.process_database_clean()
+    
+    print("\n✨ Database completely rebuilt with pristine structured data!")
+
 if __name__ == "__main__":
-    asyncio.run(main())
+    asyncio.run(rebuild_pipeline())