From 5ea7f79f56ae5e7b414ad2031af2b015362c2551 Mon Sep 17 00:00:00 2001 From: Herculino Trotta Date: Tue, 17 Nov 2026 13:01:25 -0300 Subject: [PATCH 1/2] fix(export): keep each user's names on restore and speed up big exports Restore looked categories, tags and entities up through the admin's filtered manager, so other users' transactions moved to the admin's categories and lost their tags. They are now matched to the row's owner (then to ones that owner can see) and created for that owner. Export also read tags and entities through that manager and left other users' private ones out. Recurring transactions and installment plans now export account_owner, which restore already read but never received. Transaction exports ran a few queries per row and paged prefetched rows with OFFSET, so large histories timed out. They now load relations up front and read in cursor chunks: 200k transactions export in ~43s with ~200 queries (10k took 34s and ~21k queries before). Fixes #691 Fixes #692 --- app/apps/export_app/resources/transactions.py | 53 ++++++-- app/apps/export_app/tests.py | 126 ++++++++++++++++++ app/apps/export_app/widgets/foreign_key.py | 104 +++++++++------ app/apps/export_app/widgets/many_to_many.py | 21 ++- 4 files changed, 238 insertions(+), 66 deletions(-) diff --git a/app/apps/export_app/resources/transactions.py b/app/apps/export_app/resources/transactions.py index 1abde7ca..1bc2dc99 100644 --- a/app/apps/export_app/resources/transactions.py +++ b/app/apps/export_app/resources/transactions.py @@ -1,7 +1,7 @@ from functools import reduce from operator import or_ -from django.db.models import Q +from django.db.models import Prefetch, Q from import_export import fields, resources from import_export.widgets import ForeignKeyWidget @@ -25,7 +25,22 @@ from apps.transactions.models import ( from apps.export_app.widgets.numbers import UniversalDecimalWidget -class TransactionResource(resources.ModelResource): +def with_relations(queryset, *related): + """Loads relations up front, and every owner's tags and entities, not just the exporter's.""" + return queryset.select_related(*related).prefetch_related( + Prefetch("tags", queryset=TransactionTag.all_objects.all()), + Prefetch("entities", queryset=TransactionEntity.all_objects.all()), + ) + + +class PrefetchedExportResource(resources.ModelResource): + def iter_queryset(self, queryset): + # import-export pages prefetched querysets with OFFSET, which slows down + # quadratically; Django's iterator prefetches per chunk with a cursor instead + yield from queryset.iterator(chunk_size=2000) + + +class TransactionResource(PrefetchedExportResource): account = fields.Field( attribute="account", column_name="account", @@ -64,7 +79,15 @@ class TransactionResource(resources.ModelResource): model = Transaction def get_queryset(self): - return Transaction.userless_all_objects.all() + return with_relations( + Transaction.userless_all_objects.all(), + "account", + "owner", + "category", + "installment_plan", + "recurring_transaction", + "linked_transaction", + ) class TransactionTagResource(resources.ModelResource): @@ -91,7 +114,7 @@ class TransactionCategoyResource(resources.ModelResource): return TransactionCategory.all_objects.all() -class RecurringTransactionResource(resources.ModelResource): +class RecurringTransactionResource(PrefetchedExportResource): account = fields.Field( attribute="account", column_name="account", @@ -122,18 +145,23 @@ class RecurringTransactionResource(resources.ModelResource): widget=UniversalDecimalWidget(), ) + # Lets restore find the account and its names under the right user + account_owner = fields.Field(column_name="account_owner", readonly=True) + class Meta: model = RecurringTransaction def get_queryset(self): - return RecurringTransaction.all_objects.all() + return with_relations( + RecurringTransaction.all_objects.all(), "account", "to_account", "category" + ) def dehydrate_account_owner(self, obj): """Export the account's owner ID for proper import matching.""" return obj.account.owner_id if obj.account else None -class InstallmentPlanResource(resources.ModelResource): +class InstallmentPlanResource(PrefetchedExportResource): account = fields.Field( attribute="account", column_name="account", @@ -164,11 +192,16 @@ class InstallmentPlanResource(resources.ModelResource): widget=UniversalDecimalWidget(), ) + # Lets restore find the account and its names under the right user + account_owner = fields.Field(column_name="account_owner", readonly=True) + class Meta: model = InstallmentPlan def get_queryset(self): - return InstallmentPlan.all_objects.all() + return with_relations( + InstallmentPlan.all_objects.all(), "account", "category" + ) def dehydrate_account_owner(self, obj): """Export the account's owner ID for proper import matching.""" @@ -182,7 +215,7 @@ class AnyOwnerForeignKeyWidget(ForeignKeyWidget): return self.model._base_manager.all() -class QuickTransactionResource(resources.ModelResource): +class QuickTransactionResource(PrefetchedExportResource): account = fields.Field( attribute="account", column_name="account", @@ -226,7 +259,9 @@ class QuickTransactionResource(resources.ModelResource): model = QuickTransaction def get_queryset(self): - return QuickTransaction.all_objects.all() + return with_relations( + QuickTransaction.all_objects.all(), "account", "to_account", "category", "owner" + ) SUB_TRANSACTION_BACKUP_PARENTS = ( diff --git a/app/apps/export_app/tests.py b/app/apps/export_app/tests.py index e929864c..5ea92e6c 100644 --- a/app/apps/export_app/tests.py +++ b/app/apps/export_app/tests.py @@ -4,7 +4,11 @@ import zipfile from decimal import Decimal from django.contrib.auth import get_user_model +from django.db import connection +from django.contrib.messages import get_messages from django.test import TestCase +from django.urls import reverse +from django.test.utils import CaptureQueriesContext from apps.accounts.models import Account, InvestmentClass from apps.common.middleware.thread_local import delete_current_user, write_current_user @@ -12,6 +16,9 @@ from apps.currencies.models import Currency from apps.export_app.resources.accounts import AccountResource, InvestmentClassResource from apps.export_app.resources.transactions import ( InstallmentPlanResource, + RecurringTransactionResource, + TransactionCategoyResource, + TransactionTagResource, QuickTransactionResource, SubTransactionResource, TransactionEntityResource, @@ -23,7 +30,9 @@ from apps.transactions.models import ( QuickTransaction, SubTransaction, Transaction, + TransactionCategory, TransactionEntity, + TransactionTag, ) @@ -119,3 +128,120 @@ class SubTransactionBackupTests(TestCase): def test_exports_only_the_parts_of_exported_parents(self): dataset = SubTransactionResource(parents=["installment_plan"]).export() self.assertEqual([row["name"] for row in dataset.dict], ["Principal"]) + + +class MultiUserRestoreTests(TestCase): + """Export and restore run as the admin; other users' private names must survive (#691).""" + + def setUp(self): + User = get_user_model() + self.admin = User.objects.create_superuser(email="admin@x.com", password="pw") + self.bob = User.objects.create_user(email="bob@x.com", password="pw") + usd = Currency.objects.create(code="USD", name="USD", decimal_places=2) + for user in (self.admin, self.bob): + write_current_user(user) + account = Account.objects.create(name="Checking", currency=usd) + food = TransactionCategory.objects.create(name="Food") + transaction = Transaction.objects.create( + account=account, type="EX", date=date(2030, 1, 1), amount=1, + category=food, description="lunch", + ) + transaction.tags.add(TransactionTag.objects.create(name="groceries")) + InstallmentPlan.objects.create( + account=account, type="EX", description="TV", number_of_installments=1, + start_date=date(2030, 1, 1), installment_amount=10, category=food, + ) + write_current_user(self.admin) + self.addCleanup(delete_current_user) + + def _state(self): + return sorted( + (t.account.owner.email, t.category.owner.email, [g.owner.email for g in t.tags(manager="all_objects").all()]) + for t in Transaction.userless_all_objects.all() + ) + sorted( + (p.account.owner.email, p.category.owner.email) + for p in InstallmentPlan.all_objects.all() + ) + + def test_round_trip_keeps_each_users_categories_and_tags(self): + before = self._state() + backup = io.BytesIO() + with zipfile.ZipFile(backup, "w") as z: + for resource, name in ( + (TransactionCategoyResource(), "transactions_categories"), + (TransactionTagResource(), "transactions_tags"), + (TransactionResource(), "transactions"), + (InstallmentPlanResource(), "installment_plans"), + ): + z.writestr(f"{name}.csv", resource.export().csv) + + Transaction.userless_all_objects.all().delete() + InstallmentPlan.all_objects.all().delete() + TransactionCategory.all_objects.all().delete() + TransactionTag.all_objects.all().delete() + backup.seek(0) + process_imports(None, {"zip_file": backup}) + + self.assertEqual(self._state(), before) + + def test_missing_names_are_created_for_the_rows_owner(self): + csv = TransactionResource().export().csv + Transaction.userless_all_objects.all().delete() + InstallmentPlan.all_objects.all().delete() + TransactionCategory.all_objects.all().delete() + TransactionTag.all_objects.all().delete() + + process_imports(None, {"transactions": io.BytesIO(csv.encode())}) + + self.assertEqual( + sorted(TransactionCategory.all_objects.values_list("owner__email", flat=True)), + ["admin@x.com", "bob@x.com"], + ) + self.assertEqual( + sorted(TransactionTag.all_objects.values_list("owner__email", flat=True)), + ["admin@x.com", "bob@x.com"], + ) + + def test_export_queries_do_not_grow_with_rows(self): + # A few queries per row timed big exports out (#692) + def export_queries(): + with CaptureQueriesContext(connection) as queries: + TransactionResource().export() + return len(queries) + + few = export_queries() + template = Transaction.userless_all_objects.first() + for _ in range(5): + copy = Transaction.all_objects.create( + account=template.account, type="EX", date=date(2030, 1, 2), amount=1, + category=template.category, owner=template.owner, + ) + copy.tags.set(template.tags(manager="all_objects").all()) + + self.assertEqual(export_queries(), few) + + +class ExportViewTests(TestCase): + def setUp(self): + self.client.force_login( + get_user_model().objects.create_superuser(email="a@test.com", password="pw") + ) + + def test_nothing_selected_warns_instead_of_downloading(self): + response = self.client.post(reverse("export_form"), {}, HTTP_HX_REQUEST="true") + self.assertEqual(response.status_code, 204) + self.assertEqual( + [m.level_tag for m in get_messages(response.wsgi_request)], ["warning"] + ) + + def test_non_superuser_is_refused_not_sent_back_to_login(self): + # The login page sends signed-in users back to ?next=, so redirecting there looped + self.client.force_login( + get_user_model().objects.create_user(email="b@test.com", password="pw") + ) + self.assertEqual(self.client.get(reverse("export_index")).status_code, 403) + + def test_restore_without_files_shows_the_error_in_the_form(self): + response = self.client.post(reverse("restore_form"), {}, HTTP_HX_REQUEST="true") + self.assertContains(response, "Please upload either a ZIP file or at least one CSV file") + self.assertContains(response, 'id="restore-form"') diff --git a/app/apps/export_app/widgets/foreign_key.py b/app/apps/export_app/widgets/foreign_key.py index 92463ca3..0b692b71 100644 --- a/app/apps/export_app/widgets/foreign_key.py +++ b/app/apps/export_app/widgets/foreign_key.py @@ -1,6 +1,63 @@ +from django.db.models import Q from import_export.widgets import ForeignKeyWidget +def row_owner_id(row): + """The id of the user who owns the record being restored from ``row``.""" + if not row: + return None + # Check for direct owner field first + owner_id = row.get("owner") if "owner" in row else None + # Fall back to account_owner for models like InstallmentPlan + if not owner_id and "account_owner" in row: + owner_id = row.get("account_owner") + # If still no owner, try to get it from the existing record's account + # This handles backward compatibility with older exports + if not owner_id and "id" in row and row.get("id"): + try: + from apps.transactions.models import ( + InstallmentPlan, + RecurringTransaction, + ) + + record_id = row.get("id") + for model_class in [InstallmentPlan, RecurringTransaction]: + try: + existing = model_class.all_objects.get(id=record_id) + if existing.account: + owner_id = existing.account.owner_id + break + except model_class.DoesNotExist: + continue + except Exception: + pass + # Final fallback: use the current logged-in user + # This handles restoring to a fresh database with older exports + if not owner_id: + from apps.common.middleware.thread_local import get_current_user + + user = get_current_user() + if user and user.is_authenticated: + owner_id = user.id + return owner_id or None + + +def get_or_create_for_owner(model, name, owner_id): + """ + The owner's own object named ``name``, else one they can see (public or + shared with them), else a new one they own. Restore runs as the admin, so + the user-filtered ``objects`` manager would pick the admin's objects. + """ + named = model.all_objects.filter(name=name) + return ( + named.filter(owner_id=owner_id).first() + or named.filter( + Q(visibility="public") | Q(shared_with=owner_id) | Q(owner=None) + ).first() + or model.all_objects.create(name=name, owner_id=owner_id) + ) + + class AllObjectsForeignKeyWidget(ForeignKeyWidget): """ ForeignKeyWidget that uses 'all_objects' manager for lookups, @@ -9,48 +66,10 @@ class AllObjectsForeignKeyWidget(ForeignKeyWidget): """ def get_queryset(self, value, row, *args, **kwargs): - # Use all_objects manager if available, otherwise fall back to default if hasattr(self.model, "all_objects"): qs = self.model.all_objects.all() - # Filter by owner if the row has an owner field and the model has owner - if row: - # Check for direct owner field first - owner_id = row.get("owner") if "owner" in row else None - # Fall back to account_owner for models like InstallmentPlan - if not owner_id and "account_owner" in row: - owner_id = row.get("account_owner") - # If still no owner, try to get it from the existing record's account - # This handles backward compatibility with older exports - if not owner_id and "id" in row and row.get("id"): - try: - # Try to find the existing record and get owner from its account - from apps.transactions.models import ( - InstallmentPlan, - RecurringTransaction, - ) - - record_id = row.get("id") - # Try to find the existing InstallmentPlan or RecurringTransaction - for model_class in [InstallmentPlan, RecurringTransaction]: - try: - existing = model_class.all_objects.get(id=record_id) - if existing.account: - owner_id = existing.account.owner_id - break - except model_class.DoesNotExist: - continue - except Exception: - pass - # Final fallback: use the current logged-in user - # This handles restoring to a fresh database with older exports - if not owner_id: - from apps.common.middleware.thread_local import get_current_user - - user = get_current_user() - if user and user.is_authenticated: - owner_id = user.id - if owner_id: - qs = qs.filter(owner_id=owner_id) + if owner_id := row_owner_id(row): + qs = qs.filter(owner_id=owner_id) return qs return super().get_queryset(value, row, *args, **kwargs) @@ -58,10 +77,7 @@ class AllObjectsForeignKeyWidget(ForeignKeyWidget): class AutoCreateForeignKeyWidget(ForeignKeyWidget): def clean(self, value, row=None, *args, **kwargs): if value: - try: - return super().clean(value, row, **kwargs) - except self.model.DoesNotExist: - return self.model.objects.create(name=value) + return get_or_create_for_owner(self.model, value, row_owner_id(row)) return None diff --git a/app/apps/export_app/widgets/many_to_many.py b/app/apps/export_app/widgets/many_to_many.py index b559e3a3..6aef1066 100644 --- a/app/apps/export_app/widgets/many_to_many.py +++ b/app/apps/export_app/widgets/many_to_many.py @@ -1,21 +1,16 @@ from import_export.widgets import ManyToManyWidget +from apps.export_app.widgets.foreign_key import get_or_create_for_owner, row_owner_id + class AutoCreateManyToManyWidget(ManyToManyWidget): def clean(self, value, row=None, *args, **kwargs): if not value: return [] - values = value.split(self.separator) - cleaned_values = [] - - for val in values: - val = val.strip() - if val: - try: - obj = self.model.objects.get(**{self.field: val}) - except self.model.DoesNotExist: - obj = self.model.objects.create(name=val) - cleaned_values.append(obj) - - return cleaned_values + owner_id = row_owner_id(row) + return [ + get_or_create_for_owner(self.model, val.strip(), owner_id) + for val in value.split(self.separator) + if val.strip() + ] From 8d6b86b7ca72e18def9a15ae2f30c8506cd12296 Mon Sep 17 00:00:00 2001 From: Herculino Trotta Date: Tue, 17 Nov 2026 13:01:25 -0300 Subject: [PATCH 2/2] feat(export): export and restore on the page, with progress and messages The export and restore forms now sit on the Export and Restore page instead of an offcanvas. Exporting shows a progress line and a toast when the download starts; restoring splits ZIP and CSV uploads into tabs and shows errors in the form. - An empty export warns instead of downloading the message as a file - Export errors show a message instead of a stuck spinner - The download link is kept long enough for Safari to save the file - Non-superusers get a 403 instead of a redirect loop with the login - The asset classes CSV can be uploaded on its own - Drop the export form's unknown investment_classes layout field --- app/apps/bank_sync/tests/test_bank_sync.py | 6 +- app/apps/export_app/forms.py | 129 +++-------- app/apps/export_app/views.py | 203 +++++++++--------- .../export_app/fragments/export.html | 38 +++- .../export_app/fragments/restore.html | 55 +++-- app/templates/export_app/pages/index.html | 29 +-- frontend/src/js/_htmx.js | 10 +- 7 files changed, 235 insertions(+), 235 deletions(-) diff --git a/app/apps/bank_sync/tests/test_bank_sync.py b/app/apps/bank_sync/tests/test_bank_sync.py index c9f28d9c..97bf797d 100644 --- a/app/apps/bank_sync/tests/test_bank_sync.py +++ b/app/apps/bank_sync/tests/test_bank_sync.py @@ -7,6 +7,7 @@ from decimal import Decimal from unittest.mock import MagicMock, patch from django.contrib.auth import get_user_model +from django.contrib.messages import get_messages from django.core.exceptions import ValidationError from django.core.files.uploadedfile import SimpleUploadedFile from django.test import TestCase @@ -879,7 +880,10 @@ class ExportRestoreTests(TestCase): response = self.client.post( reverse("restore_form"), {"zip_file": upload}, HTTP_HX_REQUEST="true" ) - self.assertEqual(response.status_code, 204) + self.assertIn( + "Data restored successfully", + [m.message for m in get_messages(response.wsgi_request)], + ) service = BankSyncService._base_manager.get() self.assertEqual(service.owner, self.other) diff --git a/app/apps/export_app/forms.py b/app/apps/export_app/forms.py index 7d50ffb1..b4b4e9bf 100644 --- a/app/apps/export_app/forms.py +++ b/app/apps/export_app/forms.py @@ -1,139 +1,107 @@ -from apps.common.widgets.crispy.submit import NoClassSubmit -from crispy_forms.bootstrap import FormActions -from crispy_forms.helper import FormHelper -from crispy_forms.layout import HTML, Layout from django import forms from django.utils.translation import gettext_lazy as _ class ExportForm(forms.Form): + # Rendered by export_app/fragments/export.html, in declaration order users = forms.BooleanField( required=False, - widget=forms.CheckboxInput(), + widget=forms.CheckboxInput(attrs={"class": "checkbox"}), label=_("Users"), initial=True, ) accounts = forms.BooleanField( required=False, - widget=forms.CheckboxInput(), + widget=forms.CheckboxInput(attrs={"class": "checkbox"}), label=_("Accounts"), initial=True, ) currencies = forms.BooleanField( required=False, - widget=forms.CheckboxInput(), + widget=forms.CheckboxInput(attrs={"class": "checkbox"}), label=_("Currencies"), initial=True, ) transactions = forms.BooleanField( required=False, - widget=forms.CheckboxInput(), + widget=forms.CheckboxInput(attrs={"class": "checkbox"}), label=_("Transactions"), initial=True, ) categories = forms.BooleanField( required=False, - widget=forms.CheckboxInput(), + widget=forms.CheckboxInput(attrs={"class": "checkbox"}), label=_("Categories"), initial=True, ) - tags = forms.BooleanField( - required=False, - widget=forms.CheckboxInput(), - label=_("Tags"), - initial=False, - ) entities = forms.BooleanField( required=False, - widget=forms.CheckboxInput(), + widget=forms.CheckboxInput(attrs={"class": "checkbox"}), label=_("Entities"), initial=False, ) - recurring_transactions = forms.BooleanField( + tags = forms.BooleanField( required=False, - widget=forms.CheckboxInput(), - label=_("Recurring Transactions"), - initial=True, + widget=forms.CheckboxInput(attrs={"class": "checkbox"}), + label=_("Tags"), + initial=False, ) installment_plans = forms.BooleanField( required=False, - widget=forms.CheckboxInput(), + widget=forms.CheckboxInput(attrs={"class": "checkbox"}), label=_("Installment Plans"), initial=True, ) + recurring_transactions = forms.BooleanField( + required=False, + widget=forms.CheckboxInput(attrs={"class": "checkbox"}), + label=_("Recurring Transactions"), + initial=True, + ) quick_transactions = forms.BooleanField( required=False, - widget=forms.CheckboxInput(), + widget=forms.CheckboxInput(attrs={"class": "checkbox"}), label=_("Quick Transactions"), initial=True, ) - exchange_rates = forms.BooleanField( - required=False, - widget=forms.CheckboxInput(), - label=_("Exchange Rates"), - initial=False, - ) exchange_rates_services = forms.BooleanField( required=False, - widget=forms.CheckboxInput(), + widget=forms.CheckboxInput(attrs={"class": "checkbox"}), label=_("Automatic Exchange Rates"), initial=False, ) + exchange_rates = forms.BooleanField( + required=False, + widget=forms.CheckboxInput(attrs={"class": "checkbox"}), + label=_("Exchange Rates"), + initial=False, + ) bank_sync = forms.BooleanField( required=False, - widget=forms.CheckboxInput(), + widget=forms.CheckboxInput(attrs={"class": "checkbox"}), label=_("Bank Sync"), help_text=_("Includes the providers' API keys"), initial=False, ) rules = forms.BooleanField( required=False, - widget=forms.CheckboxInput(), + widget=forms.CheckboxInput(attrs={"class": "checkbox"}), label=_("Rules"), initial=True, ) dca = forms.BooleanField( required=False, - widget=forms.CheckboxInput(), + widget=forms.CheckboxInput(attrs={"class": "checkbox"}), label=_("DCA"), initial=False, ) import_profiles = forms.BooleanField( required=False, - widget=forms.CheckboxInput(), + widget=forms.CheckboxInput(attrs={"class": "checkbox"}), label=_("Import Profiles"), initial=True, ) - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - - self.helper = FormHelper() - self.helper.form_tag = False - self.helper.form_method = "post" - self.helper.layout = Layout( - "users", - "accounts", - "investment_classes", - "currencies", - "transactions", - "categories", - "entities", - "tags", - "installment_plans", - "recurring_transactions", - "quick_transactions", - "exchange_rates_services", - "exchange_rates", - "bank_sync", - "rules", - "dca", - "import_profiles", - FormActions( - NoClassSubmit("submit", _("Export"), css_class="btn btn-primary"), - ), - ) - class RestoreForm(forms.Form): zip_file = forms.FileField( @@ -178,38 +146,11 @@ class RestoreForm(forms.Form): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) - - self.helper = FormHelper() - self.helper.form_tag = False - self.helper.form_method = "post" - self.helper.layout = Layout( - "zip_file", - HTML('
'), - "users", - "accounts", - "currencies", - "transactions", - "sub_transactions", - "transactions_categories", - "transactions_entities", - "transactions_tags", - "installment_plans", - "recurring_transactions", - "quick_transactions", - "automatic_exchange_rates", - "exchange_rates", - "bank_sync_services", - "bank_sync_accounts", - "transaction_rules", - "transaction_rules_actions", - "transaction_rules_update_or_create", - "dca_strategies", - "dca_entries", - "import_profiles", - FormActions( - NoClassSubmit("submit", _("Restore"), css_class="btn btn-primary"), - ), - ) + # Rendered by export_app/fragments/restore.html + for name, field in self.fields.items(): + field.widget.attrs.update( + {"class": "file-input w-full", "accept": ".zip" if name == "zip_file" else ".csv"} + ) def clean(self): cleaned_data = super().clean() diff --git a/app/apps/export_app/views.py b/app/apps/export_app/views.py index 3a521ddc..5cab9d08 100644 --- a/app/apps/export_app/views.py +++ b/app/apps/export_app/views.py @@ -3,7 +3,7 @@ import zipfile from io import BytesIO from django.contrib import messages -from django.contrib.auth.decorators import login_required, user_passes_test +from django.contrib.auth.decorators import login_required from django.db import transaction from django.http import HttpResponse from django.shortcuts import render @@ -51,21 +51,26 @@ from apps.export_app.resources.transactions import ( ) from apps.export_app.resources.users import UserResource from apps.common.decorators.demo import disabled_on_demo +from apps.common.decorators.user import is_superuser logger = logging.getLogger() @login_required @disabled_on_demo -@user_passes_test(lambda u: u.is_superuser) +@is_superuser @require_http_methods(["GET"]) def export_index(request): - return render(request, "export_app/pages/index.html") + return render( + request, + "export_app/pages/index.html", + {"form": ExportForm(), "restore_form": RestoreForm()}, + ) @login_required @disabled_on_demo -@user_passes_test(lambda u: u.is_superuser) +@is_superuser @require_http_methods(["GET", "POST"]) def export_form(request): timestamp = timezone.localtime(timezone.now()).strftime("%Y-%m-%dT%H-%M-%S") @@ -98,93 +103,104 @@ def export_form(request): export_dca = form.cleaned_data.get("dca", False) export_import_profiles = form.cleaned_data.get("import_profiles", False) - exports = [] - if export_users: - exports.append((UserResource().export(), "users")) - if export_accounts: - exports.append((AccountResource().export(), "accounts")) - exports.append( - (InvestmentClassResource().export(), "investment_classes") - ) - if export_currencies: - exports.append((CurrencyResource().export(), "currencies")) - if export_transactions: - exports.append((TransactionResource().export(), "transactions")) - if export_categories: - exports.append( - (TransactionCategoyResource().export(), "transactions_categories") - ) - if export_tags: - exports.append((TransactionTagResource().export(), "transactions_tags")) - if export_entities: - exports.append( - (TransactionEntityResource().export(), "transactions_entities") - ) - if export_installment_plans: - exports.append( - (InstallmentPlanResource().export(), "installment_plans") - ) - if export_recurring_transactions: - exports.append( - (RecurringTransactionResource().export(), "recurring_transactions") - ) - if export_quick_transactions: - exports.append( - (QuickTransactionResource().export(), "quick_transactions") - ) - sub_transaction_parents = [ - parent - for parent, exported in ( - ("transaction", export_transactions), - ("installment_plan", export_installment_plans), - ("recurring_transaction", export_recurring_transactions), - ("quick_transaction", export_quick_transactions), - ) - if exported - ] - if sub_transaction_parents: - exports.append( - ( - SubTransactionResource(parents=sub_transaction_parents).export(), - "sub_transactions", + try: + exports = [] + if export_users: + exports.append((UserResource().export(), "users")) + if export_accounts: + exports.append((AccountResource().export(), "accounts")) + exports.append( + (InvestmentClassResource().export(), "investment_classes") ) - ) - if export_exchange_rates_services: - exports.append( - (ExchangeRateServiceResource().export(), "automatic_exchange_rates") - ) - if export_exchange_rates: - exports.append((ExchangeRateResource().export(), "exchange_rates")) - if form.cleaned_data.get("bank_sync"): - exports.append((BankSyncServiceResource().export(), "bank_sync_services")) - exports.append((BankSyncAccountResource().export(), "bank_sync_accounts")) - if export_rules: - exports.append( - (TransactionRuleResource().export(), "transaction_rules") - ) - exports.append( - ( - TransactionRuleActionResource().export(), - "transaction_rules_actions", + if export_currencies: + exports.append((CurrencyResource().export(), "currencies")) + if export_transactions: + exports.append((TransactionResource().export(), "transactions")) + if export_categories: + exports.append( + (TransactionCategoyResource().export(), "transactions_categories") ) - ) - exports.append( - ( - UpdateOrCreateTransactionRuleResource().export(), - "transaction_rules_update_or_create", + if export_tags: + exports.append((TransactionTagResource().export(), "transactions_tags")) + if export_entities: + exports.append( + (TransactionEntityResource().export(), "transactions_entities") ) - ) - if export_dca: - exports.append((DCAStrategyResource().export(), "dca_strategies")) - exports.append( - ( - DCAEntryResource().export(), - "dca_entries", + if export_installment_plans: + exports.append( + (InstallmentPlanResource().export(), "installment_plans") ) - ) - if export_import_profiles: - exports.append((ImportProfileResource().export(), "import_profiles")) + if export_recurring_transactions: + exports.append( + (RecurringTransactionResource().export(), "recurring_transactions") + ) + if export_quick_transactions: + exports.append( + (QuickTransactionResource().export(), "quick_transactions") + ) + sub_transaction_parents = [ + parent + for parent, exported in ( + ("transaction", export_transactions), + ("installment_plan", export_installment_plans), + ("recurring_transaction", export_recurring_transactions), + ("quick_transaction", export_quick_transactions), + ) + if exported + ] + if sub_transaction_parents: + exports.append( + ( + SubTransactionResource(parents=sub_transaction_parents).export(), + "sub_transactions", + ) + ) + if export_exchange_rates_services: + exports.append( + (ExchangeRateServiceResource().export(), "automatic_exchange_rates") + ) + if export_exchange_rates: + exports.append((ExchangeRateResource().export(), "exchange_rates")) + if form.cleaned_data.get("bank_sync"): + exports.append((BankSyncServiceResource().export(), "bank_sync_services")) + exports.append((BankSyncAccountResource().export(), "bank_sync_accounts")) + if export_rules: + exports.append( + (TransactionRuleResource().export(), "transaction_rules") + ) + exports.append( + ( + TransactionRuleActionResource().export(), + "transaction_rules_actions", + ) + ) + exports.append( + ( + UpdateOrCreateTransactionRuleResource().export(), + "transaction_rules_update_or_create", + ) + ) + if export_dca: + exports.append((DCAStrategyResource().export(), "dca_strategies")) + exports.append( + ( + DCAEntryResource().export(), + "dca_entries", + ) + ) + if export_import_profiles: + exports.append((ImportProfileResource().export(), "import_profiles")) + except Exception as e: + logger.error("Error exporting", exc_info=e) + messages.error( + request, + _("There was an error exporting your data. Check the logs for more details."), + ) + return HttpResponse(status=204, headers={"HX-Trigger": "updated"}) + + if exports: + messages.success(request, _("Export ready, your download has started")) if len(exports) >= 2: with zipfile.ZipFile(zip_buffer, "w", zipfile.ZIP_DEFLATED) as zip_file: for dataset, name in exports: @@ -212,9 +228,9 @@ def export_form(request): ) return response else: - return HttpResponse( - _("You have to select at least one export"), - ) + # 204, or the download extension would save this message as a file + messages.warning(request, _("You have to select at least one export")) + return HttpResponse(status=204, headers={"HX-Trigger": "updated"}) else: form = ExportForm() @@ -225,7 +241,7 @@ def export_form(request): @only_htmx @login_required @disabled_on_demo -@user_passes_test(lambda u: u.is_superuser) +@is_superuser @require_http_methods(["GET", "POST"]) def import_form(request): if request.method == "POST": @@ -234,12 +250,7 @@ def import_form(request): try: process_imports(request, form.cleaned_data) messages.success(request, _("Data restored successfully")) - return HttpResponse( - status=204, - headers={ - "HX-Trigger": "hide_offcanvas, updated", - }, - ) + form = RestoreForm() # a fresh form clears the chosen files and old errors except Exception as e: logger.error("Error importing", exc_info=e) messages.error( diff --git a/app/templates/export_app/fragments/export.html b/app/templates/export_app/fragments/export.html index 9b3364e7..477296ee 100644 --- a/app/templates/export_app/fragments/export.html +++ b/app/templates/export_app/fragments/export.html @@ -1,13 +1,31 @@ -{% extends "extends/offcanvas.html" %} -{% load crispy_forms_tags %} {% load i18n %} -{% block title %}{% translate 'Export' %}{% endblock %} - -{% block body %} -
-
- {% crispy form %} -
+
+ {% csrf_token %} +
+ {% for field in form %} + + {% endfor %}
-{% endblock %} + + +
diff --git a/app/templates/export_app/fragments/restore.html b/app/templates/export_app/fragments/restore.html index edf6ddf3..7f3e5781 100644 --- a/app/templates/export_app/fragments/restore.html +++ b/app/templates/export_app/fragments/restore.html @@ -1,17 +1,44 @@ -{% extends "extends/offcanvas.html" %} -{% load crispy_forms_tags %} {% load i18n %} -{% block title %}{% translate 'Restore' %}{% endblock %} - -{% block body %} -
-
- {% crispy form %} -
+
+ {% csrf_token %} + {% for error in form.non_field_errors %} + + {% endfor %} +
+ +
+ +
+ +
+
+ {% for field in form %} + {% if field.name != "zip_file" %} + + {% endif %} + {% endfor %} +
+
-{% endblock %} + + +
diff --git a/app/templates/export_app/pages/index.html b/app/templates/export_app/pages/index.html index e53f230d..59d06e84 100644 --- a/app/templates/export_app/pages/index.html +++ b/app/templates/export_app/pages/index.html @@ -4,20 +4,21 @@ {% block title %}{% translate 'Export and Restore' %}{% endblock %} {% block content %} -
-
- - +
+
{% translate 'Export and Restore' %}
+
+
+
+

{% translate 'Export' %}

+ {% include "export_app/fragments/export.html" %} +
+
+
+
+

{% translate 'Restore' %}

+ {% include "export_app/fragments/restore.html" with form=restore_form %} +
+
{% endblock %} diff --git a/frontend/src/js/_htmx.js b/frontend/src/js/_htmx.js index 2094246d..917316bb 100644 --- a/frontend/src/js/_htmx.js +++ b/frontend/src/js/_htmx.js @@ -49,13 +49,11 @@ htmx.defineExtension('htmx-download', { document.body.appendChild(link); link.click(); - // Cleanup - setTimeout(() => { - URL.revokeObjectURL(url); - link.remove(); - }, 100); + // Cleanup. Safari still reads the blob after click(), revoking early truncates the file + link.remove(); + setTimeout(() => URL.revokeObjectURL(url), 60000); - } else { + } else if (xhr.status !== 204) { console.warn(`[htmx-download] Unexpected response status: ${xhr.status}`); }