diff --git a/app/apps/export_app/resources/transactions.py b/app/apps/export_app/resources/transactions.py index 1abde7ca..1bc2dc99 100644 --- a/app/apps/export_app/resources/transactions.py +++ b/app/apps/export_app/resources/transactions.py @@ -1,7 +1,7 @@ from functools import reduce from operator import or_ -from django.db.models import Q +from django.db.models import Prefetch, Q from import_export import fields, resources from import_export.widgets import ForeignKeyWidget @@ -25,7 +25,22 @@ from apps.transactions.models import ( from apps.export_app.widgets.numbers import UniversalDecimalWidget -class TransactionResource(resources.ModelResource): +def with_relations(queryset, *related): + """Loads relations up front, and every owner's tags and entities, not just the exporter's.""" + return queryset.select_related(*related).prefetch_related( + Prefetch("tags", queryset=TransactionTag.all_objects.all()), + Prefetch("entities", queryset=TransactionEntity.all_objects.all()), + ) + + +class PrefetchedExportResource(resources.ModelResource): + def iter_queryset(self, queryset): + # import-export pages prefetched querysets with OFFSET, which slows down + # quadratically; Django's iterator prefetches per chunk with a cursor instead + yield from queryset.iterator(chunk_size=2000) + + +class TransactionResource(PrefetchedExportResource): account = fields.Field( attribute="account", column_name="account", @@ -64,7 +79,15 @@ class TransactionResource(resources.ModelResource): model = Transaction def get_queryset(self): - return Transaction.userless_all_objects.all() + return with_relations( + Transaction.userless_all_objects.all(), + "account", + "owner", + "category", + "installment_plan", + "recurring_transaction", + "linked_transaction", + ) class TransactionTagResource(resources.ModelResource): @@ -91,7 +114,7 @@ class TransactionCategoyResource(resources.ModelResource): return TransactionCategory.all_objects.all() -class RecurringTransactionResource(resources.ModelResource): +class RecurringTransactionResource(PrefetchedExportResource): account = fields.Field( attribute="account", column_name="account", @@ -122,18 +145,23 @@ class RecurringTransactionResource(resources.ModelResource): widget=UniversalDecimalWidget(), ) + # Lets restore find the account and its names under the right user + account_owner = fields.Field(column_name="account_owner", readonly=True) + class Meta: model = RecurringTransaction def get_queryset(self): - return RecurringTransaction.all_objects.all() + return with_relations( + RecurringTransaction.all_objects.all(), "account", "to_account", "category" + ) def dehydrate_account_owner(self, obj): """Export the account's owner ID for proper import matching.""" return obj.account.owner_id if obj.account else None -class InstallmentPlanResource(resources.ModelResource): +class InstallmentPlanResource(PrefetchedExportResource): account = fields.Field( attribute="account", column_name="account", @@ -164,11 +192,16 @@ class InstallmentPlanResource(resources.ModelResource): widget=UniversalDecimalWidget(), ) + # Lets restore find the account and its names under the right user + account_owner = fields.Field(column_name="account_owner", readonly=True) + class Meta: model = InstallmentPlan def get_queryset(self): - return InstallmentPlan.all_objects.all() + return with_relations( + InstallmentPlan.all_objects.all(), "account", "category" + ) def dehydrate_account_owner(self, obj): """Export the account's owner ID for proper import matching.""" @@ -182,7 +215,7 @@ class AnyOwnerForeignKeyWidget(ForeignKeyWidget): return self.model._base_manager.all() -class QuickTransactionResource(resources.ModelResource): +class QuickTransactionResource(PrefetchedExportResource): account = fields.Field( attribute="account", column_name="account", @@ -226,7 +259,9 @@ class QuickTransactionResource(resources.ModelResource): model = QuickTransaction def get_queryset(self): - return QuickTransaction.all_objects.all() + return with_relations( + QuickTransaction.all_objects.all(), "account", "to_account", "category", "owner" + ) SUB_TRANSACTION_BACKUP_PARENTS = ( diff --git a/app/apps/export_app/tests.py b/app/apps/export_app/tests.py index e929864c..5ea92e6c 100644 --- a/app/apps/export_app/tests.py +++ b/app/apps/export_app/tests.py @@ -4,7 +4,11 @@ import zipfile from decimal import Decimal from django.contrib.auth import get_user_model +from django.db import connection +from django.contrib.messages import get_messages from django.test import TestCase +from django.urls import reverse +from django.test.utils import CaptureQueriesContext from apps.accounts.models import Account, InvestmentClass from apps.common.middleware.thread_local import delete_current_user, write_current_user @@ -12,6 +16,9 @@ from apps.currencies.models import Currency from apps.export_app.resources.accounts import AccountResource, InvestmentClassResource from apps.export_app.resources.transactions import ( InstallmentPlanResource, + RecurringTransactionResource, + TransactionCategoyResource, + TransactionTagResource, QuickTransactionResource, SubTransactionResource, TransactionEntityResource, @@ -23,7 +30,9 @@ from apps.transactions.models import ( QuickTransaction, SubTransaction, Transaction, + TransactionCategory, TransactionEntity, + TransactionTag, ) @@ -119,3 +128,120 @@ class SubTransactionBackupTests(TestCase): def test_exports_only_the_parts_of_exported_parents(self): dataset = SubTransactionResource(parents=["installment_plan"]).export() self.assertEqual([row["name"] for row in dataset.dict], ["Principal"]) + + +class MultiUserRestoreTests(TestCase): + """Export and restore run as the admin; other users' private names must survive (#691).""" + + def setUp(self): + User = get_user_model() + self.admin = User.objects.create_superuser(email="admin@x.com", password="pw") + self.bob = User.objects.create_user(email="bob@x.com", password="pw") + usd = Currency.objects.create(code="USD", name="USD", decimal_places=2) + for user in (self.admin, self.bob): + write_current_user(user) + account = Account.objects.create(name="Checking", currency=usd) + food = TransactionCategory.objects.create(name="Food") + transaction = Transaction.objects.create( + account=account, type="EX", date=date(2030, 1, 1), amount=1, + category=food, description="lunch", + ) + transaction.tags.add(TransactionTag.objects.create(name="groceries")) + InstallmentPlan.objects.create( + account=account, type="EX", description="TV", number_of_installments=1, + start_date=date(2030, 1, 1), installment_amount=10, category=food, + ) + write_current_user(self.admin) + self.addCleanup(delete_current_user) + + def _state(self): + return sorted( + (t.account.owner.email, t.category.owner.email, [g.owner.email for g in t.tags(manager="all_objects").all()]) + for t in Transaction.userless_all_objects.all() + ) + sorted( + (p.account.owner.email, p.category.owner.email) + for p in InstallmentPlan.all_objects.all() + ) + + def test_round_trip_keeps_each_users_categories_and_tags(self): + before = self._state() + backup = io.BytesIO() + with zipfile.ZipFile(backup, "w") as z: + for resource, name in ( + (TransactionCategoyResource(), "transactions_categories"), + (TransactionTagResource(), "transactions_tags"), + (TransactionResource(), "transactions"), + (InstallmentPlanResource(), "installment_plans"), + ): + z.writestr(f"{name}.csv", resource.export().csv) + + Transaction.userless_all_objects.all().delete() + InstallmentPlan.all_objects.all().delete() + TransactionCategory.all_objects.all().delete() + TransactionTag.all_objects.all().delete() + backup.seek(0) + process_imports(None, {"zip_file": backup}) + + self.assertEqual(self._state(), before) + + def test_missing_names_are_created_for_the_rows_owner(self): + csv = TransactionResource().export().csv + Transaction.userless_all_objects.all().delete() + InstallmentPlan.all_objects.all().delete() + TransactionCategory.all_objects.all().delete() + TransactionTag.all_objects.all().delete() + + process_imports(None, {"transactions": io.BytesIO(csv.encode())}) + + self.assertEqual( + sorted(TransactionCategory.all_objects.values_list("owner__email", flat=True)), + ["admin@x.com", "bob@x.com"], + ) + self.assertEqual( + sorted(TransactionTag.all_objects.values_list("owner__email", flat=True)), + ["admin@x.com", "bob@x.com"], + ) + + def test_export_queries_do_not_grow_with_rows(self): + # A few queries per row timed big exports out (#692) + def export_queries(): + with CaptureQueriesContext(connection) as queries: + TransactionResource().export() + return len(queries) + + few = export_queries() + template = Transaction.userless_all_objects.first() + for _ in range(5): + copy = Transaction.all_objects.create( + account=template.account, type="EX", date=date(2030, 1, 2), amount=1, + category=template.category, owner=template.owner, + ) + copy.tags.set(template.tags(manager="all_objects").all()) + + self.assertEqual(export_queries(), few) + + +class ExportViewTests(TestCase): + def setUp(self): + self.client.force_login( + get_user_model().objects.create_superuser(email="a@test.com", password="pw") + ) + + def test_nothing_selected_warns_instead_of_downloading(self): + response = self.client.post(reverse("export_form"), {}, HTTP_HX_REQUEST="true") + self.assertEqual(response.status_code, 204) + self.assertEqual( + [m.level_tag for m in get_messages(response.wsgi_request)], ["warning"] + ) + + def test_non_superuser_is_refused_not_sent_back_to_login(self): + # The login page sends signed-in users back to ?next=, so redirecting there looped + self.client.force_login( + get_user_model().objects.create_user(email="b@test.com", password="pw") + ) + self.assertEqual(self.client.get(reverse("export_index")).status_code, 403) + + def test_restore_without_files_shows_the_error_in_the_form(self): + response = self.client.post(reverse("restore_form"), {}, HTTP_HX_REQUEST="true") + self.assertContains(response, "Please upload either a ZIP file or at least one CSV file") + self.assertContains(response, 'id="restore-form"') diff --git a/app/apps/export_app/widgets/foreign_key.py b/app/apps/export_app/widgets/foreign_key.py index 92463ca3..0b692b71 100644 --- a/app/apps/export_app/widgets/foreign_key.py +++ b/app/apps/export_app/widgets/foreign_key.py @@ -1,6 +1,63 @@ +from django.db.models import Q from import_export.widgets import ForeignKeyWidget +def row_owner_id(row): + """The id of the user who owns the record being restored from ``row``.""" + if not row: + return None + # Check for direct owner field first + owner_id = row.get("owner") if "owner" in row else None + # Fall back to account_owner for models like InstallmentPlan + if not owner_id and "account_owner" in row: + owner_id = row.get("account_owner") + # If still no owner, try to get it from the existing record's account + # This handles backward compatibility with older exports + if not owner_id and "id" in row and row.get("id"): + try: + from apps.transactions.models import ( + InstallmentPlan, + RecurringTransaction, + ) + + record_id = row.get("id") + for model_class in [InstallmentPlan, RecurringTransaction]: + try: + existing = model_class.all_objects.get(id=record_id) + if existing.account: + owner_id = existing.account.owner_id + break + except model_class.DoesNotExist: + continue + except Exception: + pass + # Final fallback: use the current logged-in user + # This handles restoring to a fresh database with older exports + if not owner_id: + from apps.common.middleware.thread_local import get_current_user + + user = get_current_user() + if user and user.is_authenticated: + owner_id = user.id + return owner_id or None + + +def get_or_create_for_owner(model, name, owner_id): + """ + The owner's own object named ``name``, else one they can see (public or + shared with them), else a new one they own. Restore runs as the admin, so + the user-filtered ``objects`` manager would pick the admin's objects. + """ + named = model.all_objects.filter(name=name) + return ( + named.filter(owner_id=owner_id).first() + or named.filter( + Q(visibility="public") | Q(shared_with=owner_id) | Q(owner=None) + ).first() + or model.all_objects.create(name=name, owner_id=owner_id) + ) + + class AllObjectsForeignKeyWidget(ForeignKeyWidget): """ ForeignKeyWidget that uses 'all_objects' manager for lookups, @@ -9,48 +66,10 @@ class AllObjectsForeignKeyWidget(ForeignKeyWidget): """ def get_queryset(self, value, row, *args, **kwargs): - # Use all_objects manager if available, otherwise fall back to default if hasattr(self.model, "all_objects"): qs = self.model.all_objects.all() - # Filter by owner if the row has an owner field and the model has owner - if row: - # Check for direct owner field first - owner_id = row.get("owner") if "owner" in row else None - # Fall back to account_owner for models like InstallmentPlan - if not owner_id and "account_owner" in row: - owner_id = row.get("account_owner") - # If still no owner, try to get it from the existing record's account - # This handles backward compatibility with older exports - if not owner_id and "id" in row and row.get("id"): - try: - # Try to find the existing record and get owner from its account - from apps.transactions.models import ( - InstallmentPlan, - RecurringTransaction, - ) - - record_id = row.get("id") - # Try to find the existing InstallmentPlan or RecurringTransaction - for model_class in [InstallmentPlan, RecurringTransaction]: - try: - existing = model_class.all_objects.get(id=record_id) - if existing.account: - owner_id = existing.account.owner_id - break - except model_class.DoesNotExist: - continue - except Exception: - pass - # Final fallback: use the current logged-in user - # This handles restoring to a fresh database with older exports - if not owner_id: - from apps.common.middleware.thread_local import get_current_user - - user = get_current_user() - if user and user.is_authenticated: - owner_id = user.id - if owner_id: - qs = qs.filter(owner_id=owner_id) + if owner_id := row_owner_id(row): + qs = qs.filter(owner_id=owner_id) return qs return super().get_queryset(value, row, *args, **kwargs) @@ -58,10 +77,7 @@ class AllObjectsForeignKeyWidget(ForeignKeyWidget): class AutoCreateForeignKeyWidget(ForeignKeyWidget): def clean(self, value, row=None, *args, **kwargs): if value: - try: - return super().clean(value, row, **kwargs) - except self.model.DoesNotExist: - return self.model.objects.create(name=value) + return get_or_create_for_owner(self.model, value, row_owner_id(row)) return None diff --git a/app/apps/export_app/widgets/many_to_many.py b/app/apps/export_app/widgets/many_to_many.py index b559e3a3..6aef1066 100644 --- a/app/apps/export_app/widgets/many_to_many.py +++ b/app/apps/export_app/widgets/many_to_many.py @@ -1,21 +1,16 @@ from import_export.widgets import ManyToManyWidget +from apps.export_app.widgets.foreign_key import get_or_create_for_owner, row_owner_id + class AutoCreateManyToManyWidget(ManyToManyWidget): def clean(self, value, row=None, *args, **kwargs): if not value: return [] - values = value.split(self.separator) - cleaned_values = [] - - for val in values: - val = val.strip() - if val: - try: - obj = self.model.objects.get(**{self.field: val}) - except self.model.DoesNotExist: - obj = self.model.objects.create(name=val) - cleaned_values.append(obj) - - return cleaned_values + owner_id = row_owner_id(row) + return [ + get_or_create_for_owner(self.model, val.strip(), owner_id) + for val in value.split(self.separator) + if val.strip() + ]