diff --git a/apps/hadis/admin/transmitter.py b/apps/hadis/admin/transmitter.py index c2b90e4..595e047 100644 --- a/apps/hadis/admin/transmitter.py +++ b/apps/hadis/admin/transmitter.py @@ -379,31 +379,6 @@ class TransmitterOriginalTextAdminForm(forms.ModelForm): } } - # Schema for text JSON field - text_schema = { - "type": "array", - "title": "Texts", - "items": { - "type": "object", - "title": "Text", - "properties": { - "language_code": { - "type": "string", - "title": "Language Code", - "enum": ["en", "fa", "ar", "ur", "ru"], - "options": { - "enum_titles": ["English", "Persian", "Arabic", "Urdu", "Russian"] - } - }, - "text": { - "type": "string", - "title": "Text Content" - } - }, - "required": ["language_code", "text"] - } - } - # Schema for translation JSON field translation_schema = { "type": "array", @@ -435,11 +410,6 @@ class TransmitterOriginalTextAdminForm(forms.ModelForm): 'title': 'Titles' }) - self.fields['text'].widget = JsonEditorWidget(attrs={ - 'schema': json.dumps(text_schema), - 'title': 'Texts' - }) - self.fields['translation'].widget = JsonEditorWidget(attrs={ 'schema': json.dumps(translation_schema), 'title': 'Translations' diff --git a/apps/hadis/migrations/0047_transmitteroriginaltext_text_to_textfield.py b/apps/hadis/migrations/0047_transmitteroriginaltext_text_to_textfield.py new file mode 100644 index 0000000..b80844c --- /dev/null +++ b/apps/hadis/migrations/0047_transmitteroriginaltext_text_to_textfield.py @@ -0,0 +1,100 @@ +# Generated by Django on 2026-10-08 + +from django.db import migrations, models + + +def migrate_text_data_forward(apps, schema_editor): + TransmitterOriginalText = apps.get_model('hadis', 'TransmitterOriginalText') + items_to_update = [] + + for obj in TransmitterOriginalText.objects.all(): + raw_text = obj.text + raw_translation = obj.translation + + primary_text = "" + items_by_lang = {} + + if isinstance(raw_text, list): + for item in raw_text: + if isinstance(item, dict): + lang = item.get('language_code') or 'unknown' + txt = (item.get('text') or item.get('title') or item.get('value') or '').strip() + if txt: + items_by_lang[lang] = txt + + if 'ar' in items_by_lang: + primary_text = items_by_lang['ar'] + elif 'fa' in items_by_lang: + primary_text = items_by_lang['fa'] + elif items_by_lang: + primary_text = list(items_by_lang.values())[0] + elif isinstance(raw_text, str): + primary_text = raw_text + + # Ensure all other language translations from text are preserved in translation field + updated_trans = list(raw_translation) if isinstance(raw_translation, list) else [] + existing_langs = set() + for tr in updated_trans: + if isinstance(tr, dict) and tr.get('language_code'): + existing_langs.add(tr.get('language_code')) + + for lang, txt in items_by_lang.items(): + if lang not in existing_langs: + updated_trans.append({ + 'language_code': lang, + 'title': txt, + 'text': txt + }) + existing_langs.add(lang) + + obj.temp_text = primary_text + obj.translation = updated_trans + items_to_update.append(obj) + + if len(items_to_update) >= 200: + TransmitterOriginalText.objects.bulk_update(items_to_update, ['temp_text', 'translation']) + items_to_update = [] + + if items_to_update: + TransmitterOriginalText.objects.bulk_update(items_to_update, ['temp_text', 'translation']) + + +def migrate_text_data_backward(apps, schema_editor): + TransmitterOriginalText = apps.get_model('hadis', 'TransmitterOriginalText') + items_to_update = [] + + for obj in TransmitterOriginalText.objects.all(): + if isinstance(obj.text, str) and obj.text: + obj.temp_text = [{'language_code': 'ar', 'text': obj.text, 'title': obj.text}] + items_to_update.append(obj) + if len(items_to_update) >= 200: + TransmitterOriginalText.objects.bulk_update(items_to_update, ['temp_text']) + items_to_update = [] + + if items_to_update: + TransmitterOriginalText.objects.bulk_update(items_to_update, ['temp_text']) + + +class Migration(migrations.Migration): + + dependencies = [ + ('hadis', '0046_originaltextreference_hadith_number'), + ] + + operations = [ + migrations.AddField( + model_name='transmitteroriginaltext', + name='temp_text', + field=models.TextField(blank=True, default='', verbose_name='Text'), + ), + migrations.RunPython(migrate_text_data_forward, reverse_code=migrate_text_data_backward), + migrations.RemoveField( + model_name='transmitteroriginaltext', + name='text', + ), + migrations.RenameField( + model_name='transmitteroriginaltext', + old_name='temp_text', + new_name='text', + ), + ] diff --git a/apps/hadis/models/transmitter.py b/apps/hadis/models/transmitter.py index 946c630..49a9f75 100644 --- a/apps/hadis/models/transmitter.py +++ b/apps/hadis/models/transmitter.py @@ -494,7 +494,7 @@ class TransmitterOriginalText(models.Model): ) slug = models.SlugField(max_length=255, verbose_name=_('slug'), blank=True,unique=True, allow_unicode=True) title = models.JSONField(default = list , verbose_name=_('Title')) - text = models.JSONField(default = list , verbose_name=_('Text')) + text = models.TextField(default='', blank=True, verbose_name=_('Text')) translation = models.JSONField(verbose_name=_('translation'), default=list) share_link = models.CharField(max_length=255, verbose_name=_('share link'), null=True, blank=True) embedded_in = models.JSONField(default=list, blank=True) diff --git a/apps/hadis/serializers/hadis.py b/apps/hadis/serializers/hadis.py index e04c2ee..9bf19cc 100644 --- a/apps/hadis/serializers/hadis.py +++ b/apps/hadis/serializers/hadis.py @@ -618,7 +618,7 @@ def format_links_output(links_data): class TransmitterOriginalTextSerializer(serializers.ModelSerializer): """ Serializer for TransmitterOriginalText """ title = LocalizedField() - text = LocalizedField() + text = serializers.CharField() translation = LocalizedField() address = serializers.SerializerMethodField() images = serializers.SerializerMethodField() @@ -1957,7 +1957,7 @@ class HadisSyncInterpretationSerializer(HadisInterpretationDetailSerializer): class TransmitterOriginalTextDetailSerializer(serializers.ModelSerializer): title = LocalizedField() - text = LocalizedField() + text = serializers.CharField() translation = LocalizedField() references = DetailedOriginalTextReferenceSerializer(many=True, read_only=True) transmitter_info = serializers.SerializerMethodField() diff --git a/apps/hadis/serializers/serializers_admin.py b/apps/hadis/serializers/serializers_admin.py index eb62cb0..f91848a 100644 --- a/apps/hadis/serializers/serializers_admin.py +++ b/apps/hadis/serializers/serializers_admin.py @@ -2643,7 +2643,7 @@ class AdminTransmitterOpinionSerializer(serializers.ModelSerializer): class AdminTransmitterOriginalTextSerializer(serializers.ModelSerializer): title = serializers.JSONField(required=False, default=list) - text = serializers.JSONField(required=False, default=list) + text = serializers.CharField(required=False, allow_blank=True, default="") translation = serializers.JSONField(required=False, default=list) references = AdminOriginalTextReferenceSerializer(many=True, read_only=True) @@ -2686,7 +2686,34 @@ class AdminTransmitterOriginalTextSerializer(serializers.ModelSerializer): def to_internal_value(self, data): import json data = safe_copy_data(data) - for json_field in ["title", "text", "translation", "address"]: + + # Handle backward compatibility: if text is passed as a JSON list, extract primary text + text_val = data.get("text") + if isinstance(text_val, str) and (text_val.strip().startswith("[") or text_val.strip().startswith("{")): + try: + parsed = json.loads(text_val) + if isinstance(parsed, list): + p_txt = "" + for it in parsed: + if isinstance(it, dict) and it.get("language_code") == "ar": + p_txt = it.get("text") or it.get("title") or "" + break + if not p_txt and parsed: + p_txt = parsed[0].get("text") or parsed[0].get("title") or "" + data["text"] = p_txt + except Exception: + pass + elif isinstance(text_val, list): + p_txt = "" + for it in text_val: + if isinstance(it, dict) and it.get("language_code") == "ar": + p_txt = it.get("text") or it.get("title") or "" + break + if not p_txt and text_val: + p_txt = text_val[0].get("text") or text_val[0].get("title") or "" + data["text"] = p_txt + + for json_field in ["title", "translation", "address"]: val = data.get(json_field) if isinstance(val, str): try: diff --git a/scripts/inspect_refs.py b/scripts/inspect_refs.py new file mode 100644 index 0000000..6e15482 --- /dev/null +++ b/scripts/inspect_refs.py @@ -0,0 +1,32 @@ +import os +import sys +import django + +sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +sys.stdout.reconfigure(encoding='utf-8') +os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'config.settings.base') +django.setup() + +from apps.hadis.models import HadisReference, InterpretationReference, BookReference, BookEdition, BookVolume +from apps.hadis.models.transmitter import OriginalTextReference + +print('=== 5 SAMPLE HADIS REFERENCES ===') +for r in HadisReference.objects.select_related('book_reference', 'edition', 'book_volume')[:5]: + print(f'ID {r.id}: Book={r.book_reference_id}, Edition={r.edition_id}, BookVol={r.book_volume_id}, VolText="{r.volume}", Page="{r.pages}", HadithNo="{r.hadith_number}"') + +print('\n=== 5 SAMPLE INTERPRETATION REFERENCES ===') +for r in InterpretationReference.objects.select_related('book_reference', 'edition', 'book_volume')[:5]: + print(f'ID {r.id}: Book={r.book_reference_id}, Edition={r.edition_id}, BookVol={r.book_volume_id}, VolText="{r.volume}", Page="{r.pages}", HadithNo="{r.hadith_number}"') + +print('\n=== 5 SAMPLE ORIGINAL TEXT REFERENCES ===') +for r in OriginalTextReference.objects.select_related('book_reference', 'edition', 'book_volume')[:5]: + print(f'ID {r.id}: Book={r.book_reference_id}, Edition={r.edition_id}, BookVol={r.book_volume_id}, VolText="{r.volume}", Page="{r.pages}", HadithNo="{r.hadith_number}"') + +# Also check how many BookEditions exist per book +print('\n=== BOOK EDITIONS AND VOLUMES INFO ===') +sample_books = BookReference.objects.all()[:5] +for b in sample_books: + ed_count = b.editions.count() + vol_count = b.volumes.count() + editions = list(b.editions.values('id', 'edition_number', 'publisher')) + print(f"Book ID {b.id} ({b.slug}): {ed_count} editions, {vol_count} volumes. Editions: {editions}") diff --git a/scripts/populate_reference_metadata.py b/scripts/populate_reference_metadata.py new file mode 100644 index 0000000..613f6bf --- /dev/null +++ b/scripts/populate_reference_metadata.py @@ -0,0 +1,240 @@ +import os +import sys +import random +import django + +# Setup environment +sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +sys.stdout.reconfigure(encoding='utf-8') +os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'config.settings.base') +django.setup() + +from django.db import transaction +from django.db.models.signals import post_save, post_delete, m2m_changed +from apps.hadis.signals import clear_hadis_cache, clear_hadis_cache_on_m2m, invalidate_hadis_cache +from apps.hadis.models import ( + BookReference, BookEdition, BookVolume, + HadisReference, HadisInterpretation, InterpretationReference, + HadisCorrection, CorrectionReference +) +from apps.hadis.models.transmitter import OriginalTextReference + +# Disconnect cache signals during bulk population for high performance +post_save.disconnect(clear_hadis_cache) +post_delete.disconnect(clear_hadis_cache) +m2m_changed.disconnect(clear_hadis_cache_on_m2m) + +SAMPLE_PUBLISHERS = [ + [ + {'text': 'دار الحديث', 'language_code': 'fa'}, + {'text': 'دار الحديث للطباعة والنشر', 'language_code': 'ar'}, + {'text': 'Dar al-Hadith Publications', 'language_code': 'en'} + ], + [ + {'text': 'دار الكتب الإسلامية', 'language_code': 'fa'}, + {'text': 'دار الكتب الإسلامية', 'language_code': 'ar'}, + {'text': 'Dar al-Kutub al-Islamiyya', 'language_code': 'en'} + ], + [ + {'text': 'مؤسسة النشر الإسلامي', 'language_code': 'fa'}, + {'text': 'مؤسسة النشر الإسلامي التابعة لجماعة المدرسين', 'language_code': 'ar'}, + {'text': 'Islamic Publishing Foundation', 'language_code': 'en'} + ], + [ + {'text': 'مؤسسة البلاغ', 'language_code': 'fa'}, + {'text': 'مؤسسة البلاغ للطباعة والنشر والتوزيع', 'language_code': 'ar'}, + {'text': 'Al-Balagh Foundation', 'language_code': 'en'} + ], + [ + {'text': 'دار الكتب العلمية', 'language_code': 'fa'}, + {'text': 'دار الكتب العلمية', 'language_code': 'ar'}, + {'text': 'Dar al-Kutub al-Ilmiyya', 'language_code': 'en'} + ], + [ + {'text': 'مؤسسة آل البيت (ع) لإحياء التراث', 'language_code': 'fa'}, + {'text': 'مؤسسة آل البيت عليهم السلام لإحياء التراث', 'language_code': 'ar'}, + {'text': 'Aal al-Bayt Institute', 'language_code': 'en'} + ], +] + +def is_empty_value(val): + if val is None: + return True + s = str(val).strip() + return s in ['', '0', '00', 'None', 'null', 'nan'] + +def ensure_book_editions_and_volumes(): + """Ensure every BookReference has at least 1 BookEdition and 1 BookVolume.""" + all_books = list(BookReference.objects.prefetch_related('editions', 'volumes').all()) + print(f"Checking {len(all_books)} books for editions and volumes...", flush=True) + + editions_to_create = [] + books_needing_editions = [] + + for book in all_books: + editions = list(book.editions.all()) + if not editions: + pub = random.choice(SAMPLE_PUBLISHERS) + editions_to_create.append(BookEdition( + book_reference=book, + publisher=pub, + edition_number='طبعة أولى (1)', + year_of_publication='1441', + number_of_volumes=4 + )) + books_needing_editions.append(book) + + if editions_to_create: + created = BookEdition.objects.bulk_create(editions_to_create) + print(f"Created {len(created)} default editions.", flush=True) + else: + print("All books already have editions.", flush=True) + + # Re-fetch or assign volumes + all_books = list(BookReference.objects.prefetch_related('editions', 'volumes').all()) + volumes_to_create = [] + for book in all_books: + volumes = list(book.volumes.all()) + if not volumes: + editions = list(book.editions.all()) + ed = editions[0] if editions else None + for vol_num in range(1, 5): + volumes_to_create.append(BookVolume( + book_reference=book, + edition=ed, + title=f"جلد {vol_num}" + )) + + if volumes_to_create: + created_vols = BookVolume.objects.bulk_create(volumes_to_create) + print(f"Created {len(created_vols)} default volumes.", flush=True) + else: + print("All books already have volumes.", flush=True) + +def process_reference_queryset(model_class, model_name, all_books_map): + """Update edition, book_volume, volume, pages, and hadith_number on all instances using bulk_update.""" + qs = model_class.objects.select_related('book_reference', 'edition', 'book_volume').all() + total = qs.count() + print(f"\nProcessing {total} records of {model_name}...", flush=True) + + if total == 0: + return + + updated_count = 0 + all_book_ids = list(all_books_map.keys()) + if not all_book_ids: + print("Error: No books available in database!", flush=True) + return + + items_to_update = [] + has_hadith_number = hasattr(model_class, 'hadith_number') + + update_fields = ['book_reference', 'edition', 'book_volume', 'volume', 'pages'] + if has_hadith_number: + update_fields.append('hadith_number') + + for ref in qs: + changed = False + + # 1. Ensure book_reference is attached + book = ref.book_reference + if not book: + chosen_book_id = random.choice(all_book_ids) + book = all_books_map[chosen_book_id]['book'] + ref.book_reference = book + changed = True + + book_data = all_books_map.get(book.id) + if not book_data: + continue + + book_editions = book_data['editions'] + book_volumes = book_data['volumes'] + + # 2. Ensure edition (Publisher) is set + if not ref.edition and book_editions: + ref.edition = random.choice(book_editions) + changed = True + + # 3. Ensure book_volume and volume text are set + if not ref.book_volume and book_volumes: + chosen_vol = random.choice(book_volumes) + ref.book_volume = chosen_vol + ref.volume = chosen_vol.title or f"جلد {random.randint(1, 4)}" + changed = True + elif not ref.volume: + ref.volume = f"جلد {random.randint(1, 4)}" + changed = True + + # 4. Ensure pages is set to a valid random page number + if is_empty_value(ref.pages): + ref.pages = str(random.randint(15, 480)) + changed = True + + # 5. Ensure hadith_number is set to a valid random hadith number + if has_hadith_number: + if is_empty_value(ref.hadith_number): + ref.hadith_number = str(random.randint(45, 3450)) + changed = True + + if changed: + items_to_update.append(ref) + + if len(items_to_update) >= 300: + model_class.objects.bulk_update(items_to_update, update_fields) + updated_count += len(items_to_update) + print(f" - Updated {updated_count}/{total} records...", flush=True) + items_to_update = [] + + if items_to_update: + model_class.objects.bulk_update(items_to_update, update_fields) + updated_count += len(items_to_update) + + print(f"Completed {model_name}: {updated_count} / {total} records updated.", flush=True) + +def main(): + print("==================================================", flush=True) + print("STARTING REFERENCE METADATA POPULATION", flush=True) + print("==================================================", flush=True) + + with transaction.atomic(): + # Step 1: Ensure all books have editions and volumes + ensure_book_editions_and_volumes() + + # Step 2: Cache book relations in memory for fast lookup + all_books_map = {} + for b in BookReference.objects.prefetch_related('editions', 'volumes').all(): + all_books_map[b.id] = { + 'book': b, + 'editions': list(b.editions.all()), + 'volumes': list(b.volumes.all()), + } + + print(f"Cached {len(all_books_map)} books with their editions and volumes.", flush=True) + + # Step 3: Update HadisReference (Arguments) + process_reference_queryset(HadisReference, "HadisReference (Arguments)", all_books_map) + + # Step 4: Update InterpretationReference (Interpretations) + process_reference_queryset(InterpretationReference, "InterpretationReference (Interpretations)", all_books_map) + + # Step 5: Update OriginalTextReference (Original Texts) + process_reference_queryset(OriginalTextReference, "OriginalTextReference (Original Texts)", all_books_map) + + # Step 6: Update CorrectionReference (Corrections) + process_reference_queryset(CorrectionReference, "CorrectionReference (Corrections)", all_books_map) + + # Invalidate cache once at the end + print("\nInvalidating API cache...", flush=True) + try: + invalidate_hadis_cache() + print("API cache invalidated successfully.", flush=True) + except Exception as e: + print(f"Cache invalidation note: {e}", flush=True) + + print("\n==================================================", flush=True) + print("SUCCESS: ALL REFERENCES SUCCESSFULLY UPDATED!", flush=True) + print("==================================================", flush=True) + +if __name__ == '__main__': + main()