From 5ea7f79f56ae5e7b414ad2031af2b015362c2551 Mon Sep 17 00:00:00 2001 From: Herculino Trotta Date: Tue, 17 Nov 2026 13:01:25 -0300 Subject: [PATCH] fix(export): keep each user's names on restore and speed up big exports Restore looked categories, tags and entities up through the admin's filtered manager, so other users' transactions moved to the admin's categories and lost their tags. They are now matched to the row's owner (then to ones that owner can see) and created for that owner. Export also read tags and entities through that manager and left other users' private ones out. Recurring transactions and installment plans now export account_owner, which restore already read but never received. Transaction exports ran a few queries per row and paged prefetched rows with OFFSET, so large histories timed out. They now load relations up front and read in cursor chunks: 200k transactions export in ~43s with ~200 queries (10k took 34s and ~21k queries before). Fixes #691 Fixes #692 --- app/apps/export_app/resources/transactions.py | 53 ++++++-- app/apps/export_app/tests.py | 126 ++++++++++++++++++ app/apps/export_app/widgets/foreign_key.py | 104 +++++++++------ app/apps/export_app/widgets/many_to_many.py | 21 ++- 4 files changed, 238 insertions(+), 66 deletions(-) diff --git a/app/apps/export_app/resources/transactions.py b/app/apps/export_app/resources/transactions.py index 1abde7ca..1bc2dc99 100644 --- a/app/apps/export_app/resources/transactions.py +++ b/app/apps/export_app/resources/transactions.py @@ -1,7 +1,7 @@ from functools import reduce from operator import or_ -from django.db.models import Q +from django.db.models import Prefetch, Q from import_export import fields, resources from import_export.widgets import ForeignKeyWidget @@ -25,7 +25,22 @@ from apps.transactions.models import ( from apps.export_app.widgets.numbers import UniversalDecimalWidget -class TransactionResource(resources.ModelResource): +def with_relations(queryset, *related): + """Loads relations up front, and every owner's tags and entities, not just the exporter's.""" + return queryset.select_related(*related).prefetch_related( + Prefetch("tags", queryset=TransactionTag.all_objects.all()), + Prefetch("entities", queryset=TransactionEntity.all_objects.all()), + ) + + +class PrefetchedExportResource(resources.ModelResource): + def iter_queryset(self, queryset): + # import-export pages prefetched querysets with OFFSET, which slows down + # quadratically; Django's iterator prefetches per chunk with a cursor instead + yield from queryset.iterator(chunk_size=2000) + + +class TransactionResource(PrefetchedExportResource): account = fields.Field( attribute="account", column_name="account", @@ -64,7 +79,15 @@ class TransactionResource(resources.ModelResource): model = Transaction def get_queryset(self): - return Transaction.userless_all_objects.all() + return with_relations( + Transaction.userless_all_objects.all(), + "account", + "owner", + "category", + "installment_plan", + "recurring_transaction", + "linked_transaction", + ) class TransactionTagResource(resources.ModelResource): @@ -91,7 +114,7 @@ class TransactionCategoyResource(resources.ModelResource): return TransactionCategory.all_objects.all() -class RecurringTransactionResource(resources.ModelResource): +class RecurringTransactionResource(PrefetchedExportResource): account = fields.Field( attribute="account", column_name="account", @@ -122,18 +145,23 @@ class RecurringTransactionResource(resources.ModelResource): widget=UniversalDecimalWidget(), ) + # Lets restore find the account and its names under the right user + account_owner = fields.Field(column_name="account_owner", readonly=True) + class Meta: model = RecurringTransaction def get_queryset(self): - return RecurringTransaction.all_objects.all() + return with_relations( + RecurringTransaction.all_objects.all(), "account", "to_account", "category" + ) def dehydrate_account_owner(self, obj): """Export the account's owner ID for proper import matching.""" return obj.account.owner_id if obj.account else None -class InstallmentPlanResource(resources.ModelResource): +class InstallmentPlanResource(PrefetchedExportResource): account = fields.Field( attribute="account", column_name="account", @@ -164,11 +192,16 @@ class InstallmentPlanResource(resources.ModelResource): widget=UniversalDecimalWidget(), ) + # Lets restore find the account and its names under the right user + account_owner = fields.Field(column_name="account_owner", readonly=True) + class Meta: model = InstallmentPlan def get_queryset(self): - return InstallmentPlan.all_objects.all() + return with_relations( + InstallmentPlan.all_objects.all(), "account", "category" + ) def dehydrate_account_owner(self, obj): """Export the account's owner ID for proper import matching.""" @@ -182,7 +215,7 @@ class AnyOwnerForeignKeyWidget(ForeignKeyWidget): return self.model._base_manager.all() -class QuickTransactionResource(resources.ModelResource): +class QuickTransactionResource(PrefetchedExportResource): account = fields.Field( attribute="account", column_name="account", @@ -226,7 +259,9 @@ class QuickTransactionResource(resources.ModelResource): model = QuickTransaction def get_queryset(self): - return QuickTransaction.all_objects.all() + return with_relations( + QuickTransaction.all_objects.all(), "account", "to_account", "category", "owner" + ) SUB_TRANSACTION_BACKUP_PARENTS = ( diff --git a/app/apps/export_app/tests.py b/app/apps/export_app/tests.py index e929864c..5ea92e6c 100644 --- a/app/apps/export_app/tests.py +++ b/app/apps/export_app/tests.py @@ -4,7 +4,11 @@ import zipfile from decimal import Decimal from django.contrib.auth import get_user_model +from django.db import connection +from django.contrib.messages import get_messages from django.test import TestCase +from django.urls import reverse +from django.test.utils import CaptureQueriesContext from apps.accounts.models import Account, InvestmentClass from apps.common.middleware.thread_local import delete_current_user, write_current_user @@ -12,6 +16,9 @@ from apps.currencies.models import Currency from apps.export_app.resources.accounts import AccountResource, InvestmentClassResource from apps.export_app.resources.transactions import ( InstallmentPlanResource, + RecurringTransactionResource, + TransactionCategoyResource, + TransactionTagResource, QuickTransactionResource, SubTransactionResource, TransactionEntityResource, @@ -23,7 +30,9 @@ from apps.transactions.models import ( QuickTransaction, SubTransaction, Transaction, + TransactionCategory, TransactionEntity, + TransactionTag, ) @@ -119,3 +128,120 @@ class SubTransactionBackupTests(TestCase): def test_exports_only_the_parts_of_exported_parents(self): dataset = SubTransactionResource(parents=["installment_plan"]).export() self.assertEqual([row["name"] for row in dataset.dict], ["Principal"]) + + +class MultiUserRestoreTests(TestCase): + """Export and restore run as the admin; other users' private names must survive (#691).""" + + def setUp(self): + User = get_user_model() + self.admin = User.objects.create_superuser(email="admin@x.com", password="pw") + self.bob = User.objects.create_user(email="bob@x.com", password="pw") + usd = Currency.objects.create(code="USD", name="USD", decimal_places=2) + for user in (self.admin, self.bob): + write_current_user(user) + account = Account.objects.create(name="Checking", currency=usd) + food = TransactionCategory.objects.create(name="Food") + transaction = Transaction.objects.create( + account=account, type="EX", date=date(2030, 1, 1), amount=1, + category=food, description="lunch", + ) + transaction.tags.add(TransactionTag.objects.create(name="groceries")) + InstallmentPlan.objects.create( + account=account, type="EX", description="TV", number_of_installments=1, + start_date=date(2030, 1, 1), installment_amount=10, category=food, + ) + write_current_user(self.admin) + self.addCleanup(delete_current_user) + + def _state(self): + return sorted( + (t.account.owner.email, t.category.owner.email, [g.owner.email for g in t.tags(manager="all_objects").all()]) + for t in Transaction.userless_all_objects.all() + ) + sorted( + (p.account.owner.email, p.category.owner.email) + for p in InstallmentPlan.all_objects.all() + ) + + def test_round_trip_keeps_each_users_categories_and_tags(self): + before = self._state() + backup = io.BytesIO() + with zipfile.ZipFile(backup, "w") as z: + for resource, name in ( + (TransactionCategoyResource(), "transactions_categories"), + (TransactionTagResource(), "transactions_tags"), + (TransactionResource(), "transactions"), + (InstallmentPlanResource(), "installment_plans"), + ): + z.writestr(f"{name}.csv", resource.export().csv) + + Transaction.userless_all_objects.all().delete() + InstallmentPlan.all_objects.all().delete() + TransactionCategory.all_objects.all().delete() + TransactionTag.all_objects.all().delete() + backup.seek(0) + process_imports(None, {"zip_file": backup}) + + self.assertEqual(self._state(), before) + + def test_missing_names_are_created_for_the_rows_owner(self): + csv = TransactionResource().export().csv + Transaction.userless_all_objects.all().delete() + InstallmentPlan.all_objects.all().delete() + TransactionCategory.all_objects.all().delete() + TransactionTag.all_objects.all().delete() + + process_imports(None, {"transactions": io.BytesIO(csv.encode())}) + + self.assertEqual( + sorted(TransactionCategory.all_objects.values_list("owner__email", flat=True)), + ["admin@x.com", "bob@x.com"], + ) + self.assertEqual( + sorted(TransactionTag.all_objects.values_list("owner__email", flat=True)), + ["admin@x.com", "bob@x.com"], + ) + + def test_export_queries_do_not_grow_with_rows(self): + # A few queries per row timed big exports out (#692) + def export_queries(): + with CaptureQueriesContext(connection) as queries: + TransactionResource().export() + return len(queries) + + few = export_queries() + template = Transaction.userless_all_objects.first() + for _ in range(5): + copy = Transaction.all_objects.create( + account=template.account, type="EX", date=date(2030, 1, 2), amount=1, + category=template.category, owner=template.owner, + ) + copy.tags.set(template.tags(manager="all_objects").all()) + + self.assertEqual(export_queries(), few) + + +class ExportViewTests(TestCase): + def setUp(self): + self.client.force_login( + get_user_model().objects.create_superuser(email="a@test.com", password="pw") + ) + + def test_nothing_selected_warns_instead_of_downloading(self): + response = self.client.post(reverse("export_form"), {}, HTTP_HX_REQUEST="true") + self.assertEqual(response.status_code, 204) + self.assertEqual( + [m.level_tag for m in get_messages(response.wsgi_request)], ["warning"] + ) + + def test_non_superuser_is_refused_not_sent_back_to_login(self): + # The login page sends signed-in users back to ?next=, so redirecting there looped + self.client.force_login( + get_user_model().objects.create_user(email="b@test.com", password="pw") + ) + self.assertEqual(self.client.get(reverse("export_index")).status_code, 403) + + def test_restore_without_files_shows_the_error_in_the_form(self): + response = self.client.post(reverse("restore_form"), {}, HTTP_HX_REQUEST="true") + self.assertContains(response, "Please upload either a ZIP file or at least one CSV file") + self.assertContains(response, 'id="restore-form"') diff --git a/app/apps/export_app/widgets/foreign_key.py b/app/apps/export_app/widgets/foreign_key.py index 92463ca3..0b692b71 100644 --- a/app/apps/export_app/widgets/foreign_key.py +++ b/app/apps/export_app/widgets/foreign_key.py @@ -1,6 +1,63 @@ +from django.db.models import Q from import_export.widgets import ForeignKeyWidget +def row_owner_id(row): + """The id of the user who owns the record being restored from ``row``.""" + if not row: + return None + # Check for direct owner field first + owner_id = row.get("owner") if "owner" in row else None + # Fall back to account_owner for models like InstallmentPlan + if not owner_id and "account_owner" in row: + owner_id = row.get("account_owner") + # If still no owner, try to get it from the existing record's account + # This handles backward compatibility with older exports + if not owner_id and "id" in row and row.get("id"): + try: + from apps.transactions.models import ( + InstallmentPlan, + RecurringTransaction, + ) + + record_id = row.get("id") + for model_class in [InstallmentPlan, RecurringTransaction]: + try: + existing = model_class.all_objects.get(id=record_id) + if existing.account: + owner_id = existing.account.owner_id + break + except model_class.DoesNotExist: + continue + except Exception: + pass + # Final fallback: use the current logged-in user + # This handles restoring to a fresh database with older exports + if not owner_id: + from apps.common.middleware.thread_local import get_current_user + + user = get_current_user() + if user and user.is_authenticated: + owner_id = user.id + return owner_id or None + + +def get_or_create_for_owner(model, name, owner_id): + """ + The owner's own object named ``name``, else one they can see (public or + shared with them), else a new one they own. Restore runs as the admin, so + the user-filtered ``objects`` manager would pick the admin's objects. + """ + named = model.all_objects.filter(name=name) + return ( + named.filter(owner_id=owner_id).first() + or named.filter( + Q(visibility="public") | Q(shared_with=owner_id) | Q(owner=None) + ).first() + or model.all_objects.create(name=name, owner_id=owner_id) + ) + + class AllObjectsForeignKeyWidget(ForeignKeyWidget): """ ForeignKeyWidget that uses 'all_objects' manager for lookups, @@ -9,48 +66,10 @@ class AllObjectsForeignKeyWidget(ForeignKeyWidget): """ def get_queryset(self, value, row, *args, **kwargs): - # Use all_objects manager if available, otherwise fall back to default if hasattr(self.model, "all_objects"): qs = self.model.all_objects.all() - # Filter by owner if the row has an owner field and the model has owner - if row: - # Check for direct owner field first - owner_id = row.get("owner") if "owner" in row else None - # Fall back to account_owner for models like InstallmentPlan - if not owner_id and "account_owner" in row: - owner_id = row.get("account_owner") - # If still no owner, try to get it from the existing record's account - # This handles backward compatibility with older exports - if not owner_id and "id" in row and row.get("id"): - try: - # Try to find the existing record and get owner from its account - from apps.transactions.models import ( - InstallmentPlan, - RecurringTransaction, - ) - - record_id = row.get("id") - # Try to find the existing InstallmentPlan or RecurringTransaction - for model_class in [InstallmentPlan, RecurringTransaction]: - try: - existing = model_class.all_objects.get(id=record_id) - if existing.account: - owner_id = existing.account.owner_id - break - except model_class.DoesNotExist: - continue - except Exception: - pass - # Final fallback: use the current logged-in user - # This handles restoring to a fresh database with older exports - if not owner_id: - from apps.common.middleware.thread_local import get_current_user - - user = get_current_user() - if user and user.is_authenticated: - owner_id = user.id - if owner_id: - qs = qs.filter(owner_id=owner_id) + if owner_id := row_owner_id(row): + qs = qs.filter(owner_id=owner_id) return qs return super().get_queryset(value, row, *args, **kwargs) @@ -58,10 +77,7 @@ class AllObjectsForeignKeyWidget(ForeignKeyWidget): class AutoCreateForeignKeyWidget(ForeignKeyWidget): def clean(self, value, row=None, *args, **kwargs): if value: - try: - return super().clean(value, row, **kwargs) - except self.model.DoesNotExist: - return self.model.objects.create(name=value) + return get_or_create_for_owner(self.model, value, row_owner_id(row)) return None diff --git a/app/apps/export_app/widgets/many_to_many.py b/app/apps/export_app/widgets/many_to_many.py index b559e3a3..6aef1066 100644 --- a/app/apps/export_app/widgets/many_to_many.py +++ b/app/apps/export_app/widgets/many_to_many.py @@ -1,21 +1,16 @@ from import_export.widgets import ManyToManyWidget +from apps.export_app.widgets.foreign_key import get_or_create_for_owner, row_owner_id + class AutoCreateManyToManyWidget(ManyToManyWidget): def clean(self, value, row=None, *args, **kwargs): if not value: return [] - values = value.split(self.separator) - cleaned_values = [] - - for val in values: - val = val.strip() - if val: - try: - obj = self.model.objects.get(**{self.field: val}) - except self.model.DoesNotExist: - obj = self.model.objects.create(name=val) - cleaned_values.append(obj) - - return cleaned_values + owner_id = row_owner_id(row) + return [ + get_or_create_for_owner(self.model, val.strip(), owner_id) + for val in value.split(self.separator) + if val.strip() + ]