fix(export): keep each user's names on restore and speed up big exports

Restore looked categories, tags and entities up through the admin's
filtered manager, so other users' transactions moved to the admin's
categories and lost their tags. They are now matched to the row's
owner (then to ones that owner can see) and created for that owner.
Export also read tags and entities through that manager and left other
users' private ones out.

Recurring transactions and installment plans now export account_owner,
which restore already read but never received.

Transaction exports ran a few queries per row and paged prefetched rows
with OFFSET, so large histories timed out. They now load relations up
front and read in cursor chunks: 200k transactions export in ~43s with
~200 queries (10k took 34s and ~21k queries before).

Fixes #691
Fixes #692
This commit is contained in:
Herculino Trotta committed 2026-11-17 13:01:25 -03:00
1 parent 75c854d034
commit 5ea7f79f56
4 files changed
+238 -66

No files matched your search

+44 -9
View File
@@ -1,7 +1,7 @@
from functools import reduce
from operator import or_
from django.db.models import Q
from django.db.models import Prefetch, Q
from import_export import fields, resources
from import_export.widgets import ForeignKeyWidget
@@ -25,7 +25,22 @@ from apps.transactions.models import (
from apps.export_app.widgets.numbers import UniversalDecimalWidget
class TransactionResource(resources.ModelResource):
def with_relations(queryset, *related):
"""Loads relations up front, and every owner's tags and entities, not just the exporter's."""
return queryset.select_related(*related).prefetch_related(
Prefetch("tags", queryset=TransactionTag.all_objects.all()),
Prefetch("entities", queryset=TransactionEntity.all_objects.all()),
)
class PrefetchedExportResource(resources.ModelResource):
def iter_queryset(self, queryset):
# import-export pages prefetched querysets with OFFSET, which slows down
# quadratically; Django's iterator prefetches per chunk with a cursor instead
yield from queryset.iterator(chunk_size=2000)
class TransactionResource(PrefetchedExportResource):
account = fields.Field(
attribute="account",
column_name="account",
@@ -64,7 +79,15 @@ class TransactionResource(resources.ModelResource):
model = Transaction
def get_queryset(self):
return Transaction.userless_all_objects.all()
return with_relations(
Transaction.userless_all_objects.all(),
"account",
"owner",
"category",
"installment_plan",
"recurring_transaction",
"linked_transaction",
)
class TransactionTagResource(resources.ModelResource):
@@ -91,7 +114,7 @@ class TransactionCategoyResource(resources.ModelResource):
return TransactionCategory.all_objects.all()
class RecurringTransactionResource(resources.ModelResource):
class RecurringTransactionResource(PrefetchedExportResource):
account = fields.Field(
attribute="account",
column_name="account",
@@ -122,18 +145,23 @@ class RecurringTransactionResource(resources.ModelResource):
widget=UniversalDecimalWidget(),
)
# Lets restore find the account and its names under the right user
account_owner = fields.Field(column_name="account_owner", readonly=True)
class Meta:
model = RecurringTransaction
def get_queryset(self):
return RecurringTransaction.all_objects.all()
return with_relations(
RecurringTransaction.all_objects.all(), "account", "to_account", "category"
)
def dehydrate_account_owner(self, obj):
"""Export the account's owner ID for proper import matching."""
return obj.account.owner_id if obj.account else None
class InstallmentPlanResource(resources.ModelResource):
class InstallmentPlanResource(PrefetchedExportResource):
account = fields.Field(
attribute="account",
column_name="account",
@@ -164,11 +192,16 @@ class InstallmentPlanResource(resources.ModelResource):
widget=UniversalDecimalWidget(),
)
# Lets restore find the account and its names under the right user
account_owner = fields.Field(column_name="account_owner", readonly=True)
class Meta:
model = InstallmentPlan
def get_queryset(self):
return InstallmentPlan.all_objects.all()
return with_relations(
InstallmentPlan.all_objects.all(), "account", "category"
)
def dehydrate_account_owner(self, obj):
"""Export the account's owner ID for proper import matching."""
@@ -182,7 +215,7 @@ class AnyOwnerForeignKeyWidget(ForeignKeyWidget):
return self.model._base_manager.all()
class QuickTransactionResource(resources.ModelResource):
class QuickTransactionResource(PrefetchedExportResource):
account = fields.Field(
attribute="account",
column_name="account",
@@ -226,7 +259,9 @@ class QuickTransactionResource(resources.ModelResource):
model = QuickTransaction
def get_queryset(self):
return QuickTransaction.all_objects.all()
return with_relations(
QuickTransaction.all_objects.all(), "account", "to_account", "category", "owner"
)
SUB_TRANSACTION_BACKUP_PARENTS = (
+126
View File
@@ -4,7 +4,11 @@ import zipfile
from decimal import Decimal
from django.contrib.auth import get_user_model
from django.db import connection
from django.contrib.messages import get_messages
from django.test import TestCase
from django.urls import reverse
from django.test.utils import CaptureQueriesContext
from apps.accounts.models import Account, InvestmentClass
from apps.common.middleware.thread_local import delete_current_user, write_current_user
@@ -12,6 +16,9 @@ from apps.currencies.models import Currency
from apps.export_app.resources.accounts import AccountResource, InvestmentClassResource
from apps.export_app.resources.transactions import (
InstallmentPlanResource,
RecurringTransactionResource,
TransactionCategoyResource,
TransactionTagResource,
QuickTransactionResource,
SubTransactionResource,
TransactionEntityResource,
@@ -23,7 +30,9 @@ from apps.transactions.models import (
QuickTransaction,
SubTransaction,
Transaction,
TransactionCategory,
TransactionEntity,
TransactionTag,
)
@@ -119,3 +128,120 @@ class SubTransactionBackupTests(TestCase):
def test_exports_only_the_parts_of_exported_parents(self):
dataset = SubTransactionResource(parents=["installment_plan"]).export()
self.assertEqual([row["name"] for row in dataset.dict], ["Principal"])
class MultiUserRestoreTests(TestCase):
"""Export and restore run as the admin; other users' private names must survive (#691)."""
def setUp(self):
User = get_user_model()
self.admin = User.objects.create_superuser(email="admin@x.com", password="pw")
self.bob = User.objects.create_user(email="bob@x.com", password="pw")
usd = Currency.objects.create(code="USD", name="USD", decimal_places=2)
for user in (self.admin, self.bob):
write_current_user(user)
account = Account.objects.create(name="Checking", currency=usd)
food = TransactionCategory.objects.create(name="Food")
transaction = Transaction.objects.create(
account=account, type="EX", date=date(2030, 1, 1), amount=1,
category=food, description="lunch",
)
transaction.tags.add(TransactionTag.objects.create(name="groceries"))
InstallmentPlan.objects.create(
account=account, type="EX", description="TV", number_of_installments=1,
start_date=date(2030, 1, 1), installment_amount=10, category=food,
)
write_current_user(self.admin)
self.addCleanup(delete_current_user)
def _state(self):
return sorted(
(t.account.owner.email, t.category.owner.email, [g.owner.email for g in t.tags(manager="all_objects").all()])
for t in Transaction.userless_all_objects.all()
) + sorted(
(p.account.owner.email, p.category.owner.email)
for p in InstallmentPlan.all_objects.all()
)
def test_round_trip_keeps_each_users_categories_and_tags(self):
before = self._state()
backup = io.BytesIO()
with zipfile.ZipFile(backup, "w") as z:
for resource, name in (
(TransactionCategoyResource(), "transactions_categories"),
(TransactionTagResource(), "transactions_tags"),
(TransactionResource(), "transactions"),
(InstallmentPlanResource(), "installment_plans"),
):
z.writestr(f"{name}.csv", resource.export().csv)
Transaction.userless_all_objects.all().delete()
InstallmentPlan.all_objects.all().delete()
TransactionCategory.all_objects.all().delete()
TransactionTag.all_objects.all().delete()
backup.seek(0)
process_imports(None, {"zip_file": backup})
self.assertEqual(self._state(), before)
def test_missing_names_are_created_for_the_rows_owner(self):
csv = TransactionResource().export().csv
Transaction.userless_all_objects.all().delete()
InstallmentPlan.all_objects.all().delete()
TransactionCategory.all_objects.all().delete()
TransactionTag.all_objects.all().delete()
process_imports(None, {"transactions": io.BytesIO(csv.encode())})
self.assertEqual(
sorted(TransactionCategory.all_objects.values_list("owner__email", flat=True)),
["admin@x.com", "bob@x.com"],
)
self.assertEqual(
sorted(TransactionTag.all_objects.values_list("owner__email", flat=True)),
["admin@x.com", "bob@x.com"],
)
def test_export_queries_do_not_grow_with_rows(self):
# A few queries per row timed big exports out (#692)
def export_queries():
with CaptureQueriesContext(connection) as queries:
TransactionResource().export()
return len(queries)
few = export_queries()
template = Transaction.userless_all_objects.first()
for _ in range(5):
copy = Transaction.all_objects.create(
account=template.account, type="EX", date=date(2030, 1, 2), amount=1,
category=template.category, owner=template.owner,
)
copy.tags.set(template.tags(manager="all_objects").all())
self.assertEqual(export_queries(), few)
class ExportViewTests(TestCase):
def setUp(self):
self.client.force_login(
get_user_model().objects.create_superuser(email="a@test.com", password="pw")
)
def test_nothing_selected_warns_instead_of_downloading(self):
response = self.client.post(reverse("export_form"), {}, HTTP_HX_REQUEST="true")
self.assertEqual(response.status_code, 204)
self.assertEqual(
[m.level_tag for m in get_messages(response.wsgi_request)], ["warning"]
)
def test_non_superuser_is_refused_not_sent_back_to_login(self):
# The login page sends signed-in users back to ?next=, so redirecting there looped
self.client.force_login(
get_user_model().objects.create_user(email="b@test.com", password="pw")
)
self.assertEqual(self.client.get(reverse("export_index")).status_code, 403)
def test_restore_without_files_shows_the_error_in_the_form(self):
response = self.client.post(reverse("restore_form"), {}, HTTP_HX_REQUEST="true")
self.assertContains(response, "Please upload either a ZIP file or at least one CSV file")
self.assertContains(response, 'id="restore-form"')
+60 -44
View File
@@ -1,6 +1,63 @@
from django.db.models import Q
from import_export.widgets import ForeignKeyWidget
def row_owner_id(row):
"""The id of the user who owns the record being restored from ``row``."""
if not row:
return None
# Check for direct owner field first
owner_id = row.get("owner") if "owner" in row else None
# Fall back to account_owner for models like InstallmentPlan
if not owner_id and "account_owner" in row:
owner_id = row.get("account_owner")
# If still no owner, try to get it from the existing record's account
# This handles backward compatibility with older exports
if not owner_id and "id" in row and row.get("id"):
try:
from apps.transactions.models import (
InstallmentPlan,
RecurringTransaction,
)
record_id = row.get("id")
for model_class in [InstallmentPlan, RecurringTransaction]:
try:
existing = model_class.all_objects.get(id=record_id)
if existing.account:
owner_id = existing.account.owner_id
break
except model_class.DoesNotExist:
continue
except Exception:
pass
# Final fallback: use the current logged-in user
# This handles restoring to a fresh database with older exports
if not owner_id:
from apps.common.middleware.thread_local import get_current_user
user = get_current_user()
if user and user.is_authenticated:
owner_id = user.id
return owner_id or None
def get_or_create_for_owner(model, name, owner_id):
"""
The owner's own object named ``name``, else one they can see (public or
shared with them), else a new one they own. Restore runs as the admin, so
the user-filtered ``objects`` manager would pick the admin's objects.
"""
named = model.all_objects.filter(name=name)
return (
named.filter(owner_id=owner_id).first()
or named.filter(
Q(visibility="public") | Q(shared_with=owner_id) | Q(owner=None)
).first()
or model.all_objects.create(name=name, owner_id=owner_id)
)
class AllObjectsForeignKeyWidget(ForeignKeyWidget):
"""
ForeignKeyWidget that uses 'all_objects' manager for lookups,
@@ -9,48 +66,10 @@ class AllObjectsForeignKeyWidget(ForeignKeyWidget):
"""
def get_queryset(self, value, row, *args, **kwargs):
# Use all_objects manager if available, otherwise fall back to default
if hasattr(self.model, "all_objects"):
qs = self.model.all_objects.all()
# Filter by owner if the row has an owner field and the model has owner
if row:
# Check for direct owner field first
owner_id = row.get("owner") if "owner" in row else None
# Fall back to account_owner for models like InstallmentPlan
if not owner_id and "account_owner" in row:
owner_id = row.get("account_owner")
# If still no owner, try to get it from the existing record's account
# This handles backward compatibility with older exports
if not owner_id and "id" in row and row.get("id"):
try:
# Try to find the existing record and get owner from its account
from apps.transactions.models import (
InstallmentPlan,
RecurringTransaction,
)
record_id = row.get("id")
# Try to find the existing InstallmentPlan or RecurringTransaction
for model_class in [InstallmentPlan, RecurringTransaction]:
try:
existing = model_class.all_objects.get(id=record_id)
if existing.account:
owner_id = existing.account.owner_id
break
except model_class.DoesNotExist:
continue
except Exception:
pass
# Final fallback: use the current logged-in user
# This handles restoring to a fresh database with older exports
if not owner_id:
from apps.common.middleware.thread_local import get_current_user
user = get_current_user()
if user and user.is_authenticated:
owner_id = user.id
if owner_id:
qs = qs.filter(owner_id=owner_id)
if owner_id := row_owner_id(row):
qs = qs.filter(owner_id=owner_id)
return qs
return super().get_queryset(value, row, *args, **kwargs)
@@ -58,10 +77,7 @@ class AllObjectsForeignKeyWidget(ForeignKeyWidget):
class AutoCreateForeignKeyWidget(ForeignKeyWidget):
def clean(self, value, row=None, *args, **kwargs):
if value:
try:
return super().clean(value, row, **kwargs)
except self.model.DoesNotExist:
return self.model.objects.create(name=value)
return get_or_create_for_owner(self.model, value, row_owner_id(row))
return None
+8 -13
View File
@@ -1,21 +1,16 @@
from import_export.widgets import ManyToManyWidget
from apps.export_app.widgets.foreign_key import get_or_create_for_owner, row_owner_id
class AutoCreateManyToManyWidget(ManyToManyWidget):
def clean(self, value, row=None, *args, **kwargs):
if not value:
return []
values = value.split(self.separator)
cleaned_values = []
for val in values:
val = val.strip()
if val:
try:
obj = self.model.objects.get(**{self.field: val})
except self.model.DoesNotExist:
obj = self.model.objects.create(name=val)
cleaned_values.append(obj)
return cleaned_values
owner_id = row_owner_id(row)
return [
get_or_create_for_owner(self.model, val.strip(), owner_id)
for val in value.split(self.separator)
if val.strip()
]