diff --git a/app/apps/export_app/forms.py b/app/apps/export_app/forms.py index 69d36b5..bc9569e 100644 --- a/app/apps/export_app/forms.py +++ b/app/apps/export_app/forms.py @@ -1,6 +1,6 @@ from crispy_forms.bootstrap import FormActions from crispy_forms.helper import FormHelper -from crispy_forms.layout import Layout +from crispy_forms.layout import Layout, HTML from django import forms from django.utils.translation import gettext_lazy as _ @@ -44,6 +44,18 @@ class ExportForm(forms.Form): label=_("Entities"), initial=False, ) + recurring_transactions = forms.BooleanField( + required=False, + widget=forms.CheckboxInput(), + label=_("Recurring Transactions"), + initial=True, + ) + installment_plans = forms.BooleanField( + required=False, + widget=forms.CheckboxInput(), + label=_("Installment Planss"), + initial=True, + ) exchange_rates = forms.BooleanField( required=False, widget=forms.CheckboxInput(), @@ -56,6 +68,24 @@ class ExportForm(forms.Form): label=_("Automatic Exchange Rates"), initial=False, ) + rules = forms.BooleanField( + required=False, + widget=forms.CheckboxInput(), + label=_("Rules"), + initial=True, + ) + dca = forms.BooleanField( + required=False, + widget=forms.CheckboxInput(), + label=_("DCA"), + initial=False, + ) + import_profiles = forms.BooleanField( + required=False, + widget=forms.CheckboxInput(), + label=_("Import Profiles"), + initial=True, + ) def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) @@ -70,11 +100,90 @@ class ExportForm(forms.Form): "categories", "entities", "tags", + "installment_plans", + "recurring_transactions", "exchange_rates_services", "exchange_rates", + "rules", + "dca", + "import_profiles", FormActions( NoClassSubmit( "submit", _("Export"), css_class="btn btn-outline-primary w-100" ), ), ) + + +class RestoreForm(forms.Form): + zip_file = forms.FileField( + required=False, + help_text=_("Import a ZIP file exported from WYGIWYH"), + label=_("ZIP File"), + ) + accounts = forms.FileField(required=False, label=_("Accounts")) + currencies = forms.FileField(required=False, label=_("Currencies")) + transactions_categories = forms.FileField(required=False, label=_("Categories")) + transactions_tags = forms.FileField(required=False, label=_("Tags")) + transactions_entities = forms.FileField(required=False, label=_("Entities")) + transactions = forms.FileField(required=False, label=_("Transactions")) + installment_plans = forms.FileField(required=False, label=_("Installment Plans")) + recurring_transactions = forms.FileField( + required=False, label=_("Recurring Transactions") + ) + automatic_exchange_rates = forms.FileField( + required=False, label=_("Automatic Exchange Rates") + ) + exchange_rates = forms.FileField(required=False, label=_("Exchange Rates")) + transaction_rules = forms.FileField(required=False, label=_("Transaction rules")) + transaction_rules_actions = forms.FileField( + required=False, label=_("Edit transaction action") + ) + transaction_rules_update_or_create = forms.FileField( + required=False, label=_("Update or create transaction actions") + ) + dca_strategies = forms.FileField(required=False, label=_("DCA Strategies")) + dca_entries = forms.FileField(required=False, label=_("DCA Entries")) + import_profiles = forms.FileField(required=False, label=_("Import Profiles")) + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + self.helper = FormHelper() + self.helper.form_tag = False + self.helper.form_method = "post" + self.helper.layout = Layout( + "zip_file", + HTML("
"), + "accounts", + "currencies", + "transactions", + "transactions_categories", + "transactions_entities", + "transactions_tags", + "installment_plans", + "recurring_transactions", + "automatic_exchange_rates", + "exchange_rates", + "transaction_rules", + "transaction_rules_actions", + "transaction_rules_update_or_create", + "dca_strategies", + "dca_entries", + "import_profiles", + FormActions( + NoClassSubmit( + "submit", _("Restore"), css_class="btn btn-outline-primary w-100" + ), + ), + ) + + def clean(self): + cleaned_data = super().clean() + if not cleaned_data.get("zip_file") and not any( + cleaned_data.get(field) for field in self.fields if field != "zip_file" + ): + raise forms.ValidationError( + _("Please upload either a ZIP file or at least one CSV file") + ) + return cleaned_data diff --git a/app/apps/export_app/resources/accounts.py b/app/apps/export_app/resources/accounts.py index e2c6cd0..a1b8eb3 100644 --- a/app/apps/export_app/resources/accounts.py +++ b/app/apps/export_app/resources/accounts.py @@ -1,24 +1,25 @@ from import_export import fields, resources, widgets -from apps.accounts.models import Account +from apps.accounts.models import Account, AccountGroup from apps.export_app.widgets.foreign_key import AutoCreateForeignKeyWidget +from apps.currencies.models import Currency class AccountResource(resources.ModelResource): group = fields.Field( attribute="group", column_name="group", - widget=AutoCreateForeignKeyWidget("accounts.AccountGroup", "name"), + widget=AutoCreateForeignKeyWidget(AccountGroup, "name"), ) currency = fields.Field( attribute="currency", column_name="currency", - widget=widgets.ForeignKeyWidget("currencies.Currency", "name"), + widget=widgets.ForeignKeyWidget(Currency, "name"), ) exchange_currency = fields.Field( attribute="exchange_currency", column_name="exchange_currency", - widget=widgets.ForeignKeyWidget("currencies.Currency", "name"), + widget=widgets.ForeignKeyWidget(Currency, "name"), ) class Meta: diff --git a/app/apps/export_app/resources/currencies.py b/app/apps/export_app/resources/currencies.py index 698a528..c74ea95 100644 --- a/app/apps/export_app/resources/currencies.py +++ b/app/apps/export_app/resources/currencies.py @@ -1,5 +1,6 @@ from import_export import fields, resources, widgets +from apps.accounts.models import Account from apps.currencies.models import Currency, ExchangeRate, ExchangeRateService @@ -7,7 +8,7 @@ class CurrencyResource(resources.ModelResource): exchange_currency = fields.Field( attribute="exchange_currency", column_name="exchange_currency", - widget=widgets.ForeignKeyWidget("currencies.Currency", "name"), + widget=widgets.ForeignKeyWidget(Currency, "name"), ) class Meta: @@ -18,12 +19,12 @@ class ExchangeRateResource(resources.ModelResource): from_currency = fields.Field( attribute="from_currency", column_name="from_currency", - widget=widgets.ForeignKeyWidget("currencies.Currency", "name"), + widget=widgets.ForeignKeyWidget(Currency, "name"), ) to_currency = fields.Field( attribute="to_currency", column_name="to_currency", - widget=widgets.ForeignKeyWidget("currencies.Currency", "name"), + widget=widgets.ForeignKeyWidget(Currency, "name"), ) class Meta: @@ -34,12 +35,12 @@ class ExchangeRateServiceResource(resources.ModelResource): target_currencies = fields.Field( attribute="target_currencies", column_name="target_currencies", - widget=widgets.ManyToManyWidget("currencies.Currency", field="name"), + widget=widgets.ManyToManyWidget(Currency, field="name"), ) target_accounts = fields.Field( attribute="target_accounts", column_name="target_accounts", - widget=widgets.ForeignKeyWidget("accounts.Account", field="name"), + widget=widgets.ManyToManyWidget(Account, field="name"), ) class Meta: diff --git a/app/apps/export_app/resources/dca.py b/app/apps/export_app/resources/dca.py new file mode 100644 index 0000000..e9345cb --- /dev/null +++ b/app/apps/export_app/resources/dca.py @@ -0,0 +1,26 @@ +from import_export import fields, resources +from import_export.widgets import ForeignKeyWidget + +from apps.dca.models import DCAStrategy, DCAEntry +from apps.currencies.models import Currency + + +class DCAStrategyResource(resources.ModelResource): + target_currency = fields.Field( + attribute="target_currency", + column_name="target_currency", + widget=ForeignKeyWidget(Currency, "name"), + ) + payment_currency = fields.Field( + attribute="payment_currency", + column_name="payment_currency", + widget=ForeignKeyWidget(Currency, "name"), + ) + + class Meta: + model = DCAStrategy + + +class DCAEntryResource(resources.ModelResource): + class Meta: + model = DCAEntry diff --git a/app/apps/export_app/resources/import_app.py b/app/apps/export_app/resources/import_app.py new file mode 100644 index 0000000..043bfbd --- /dev/null +++ b/app/apps/export_app/resources/import_app.py @@ -0,0 +1,8 @@ +from import_export import resources + +from apps.import_app.models import ImportProfile + + +class ImportProfileResource(resources.ModelResource): + class Meta: + model = ImportProfile diff --git a/app/apps/export_app/resources/rules.py b/app/apps/export_app/resources/rules.py new file mode 100644 index 0000000..dcc549e --- /dev/null +++ b/app/apps/export_app/resources/rules.py @@ -0,0 +1,25 @@ +from import_export import fields, resources +from import_export.widgets import ForeignKeyWidget + +from apps.export_app.widgets.foreign_key import AutoCreateForeignKeyWidget +from apps.export_app.widgets.many_to_many import AutoCreateManyToManyWidget +from apps.rules.models import ( + TransactionRule, + TransactionRuleAction, + UpdateOrCreateTransactionRuleAction, +) + + +class TransactionRuleResource(resources.ModelResource): + class Meta: + model = TransactionRule + + +class TransactionRuleActionResource(resources.ModelResource): + class Meta: + model = TransactionRuleAction + + +class UpdateOrCreateTransactionRuleResource(resources.ModelResource): + class Meta: + model = UpdateOrCreateTransactionRuleAction diff --git a/app/apps/export_app/resources/transactions.py b/app/apps/export_app/resources/transactions.py index eb39c04..9644cd2 100644 --- a/app/apps/export_app/resources/transactions.py +++ b/app/apps/export_app/resources/transactions.py @@ -1,13 +1,17 @@ from import_export import fields, resources from import_export.widgets import ForeignKeyWidget +from apps.accounts.models import Account from apps.export_app.widgets.foreign_key import AutoCreateForeignKeyWidget from apps.export_app.widgets.many_to_many import AutoCreateManyToManyWidget +from apps.export_app.widgets.string import EmptyStringToNoneField from apps.transactions.models import ( Transaction, TransactionCategory, TransactionTag, TransactionEntity, + RecurringTransaction, + InstallmentPlan, ) @@ -15,7 +19,58 @@ class TransactionResource(resources.ModelResource): account = fields.Field( attribute="account", column_name="account", - widget=ForeignKeyWidget("accounts.Account", "name"), + widget=ForeignKeyWidget(Account, "name"), + ) + + category = fields.Field( + attribute="category", + column_name="category", + widget=AutoCreateForeignKeyWidget(TransactionCategory, "name"), + ) + + tags = fields.Field( + attribute="tags", + column_name="tags", + widget=AutoCreateManyToManyWidget(TransactionTag, field="name"), + ) + + entities = fields.Field( + attribute="entities", + column_name="entities", + widget=AutoCreateManyToManyWidget(TransactionEntity, field="name"), + ) + + internal_id = EmptyStringToNoneField( + column_name="internal_id", attribute="internal_id" + ) + + class Meta: + model = Transaction + + def get_queryset(self): + return Transaction.all_objects.all() + + +class TransactionTagResource(resources.ModelResource): + class Meta: + model = TransactionTag + + +class TransactionEntityResource(resources.ModelResource): + class Meta: + model = TransactionEntity + + +class TransactionCategoyResource(resources.ModelResource): + class Meta: + model = TransactionCategory + + +class RecurringTransactionResource(resources.ModelResource): + account = fields.Field( + attribute="account", + column_name="account", + widget=ForeignKeyWidget(Account, "name"), ) category = fields.Field( @@ -37,19 +92,33 @@ class TransactionResource(resources.ModelResource): ) class Meta: - model = Transaction + model = RecurringTransaction -class TransactionTagResource(resources.ModelResource): +class InstallmentPlanResource(resources.ModelResource): + account = fields.Field( + attribute="account", + column_name="account", + widget=ForeignKeyWidget(Account, "name"), + ) + + category = fields.Field( + attribute="category", + column_name="category", + widget=AutoCreateForeignKeyWidget(TransactionCategory, "name"), + ) + + tags = fields.Field( + attribute="tags", + column_name="tags", + widget=AutoCreateManyToManyWidget(TransactionTag, field="name"), + ) + + entities = fields.Field( + attribute="entities", + column_name="entities", + widget=AutoCreateManyToManyWidget(TransactionEntity, field="name"), + ) + class Meta: - model = TransactionTag - - -class TransactionEntityResource(resources.ModelResource): - class Meta: - model = TransactionEntity - - -class TransactionCategoyResource(resources.ModelResource): - class Meta: - model = TransactionCategory + model = InstallmentPlan diff --git a/app/apps/export_app/urls.py b/app/apps/export_app/urls.py index 77a292a..5b43a85 100644 --- a/app/apps/export_app/urls.py +++ b/app/apps/export_app/urls.py @@ -3,5 +3,6 @@ import apps.export_app.views as views urlpatterns = [ path("export/", views.export_index, name="export_index"), - path("export/export/", views.export_form, name="export_form"), + path("export/form/", views.export_form, name="export_form"), + path("export/restore/", views.import_form, name="restore_form"), ] diff --git a/app/apps/export_app/views.py b/app/apps/export_app/views.py index c8a7b33..3bfc0e5 100644 --- a/app/apps/export_app/views.py +++ b/app/apps/export_app/views.py @@ -1,31 +1,61 @@ +import logging import zipfile -from io import BytesIO +from io import BytesIO, TextIOWrapper +from django.contrib import messages +from django.contrib.auth.decorators import login_required +from django.db import transaction from django.http import HttpResponse from django.shortcuts import render +from django.utils import timezone +from django.utils.translation import gettext_lazy as _ +from django.views.decorators.http import require_http_methods +from tablib import Dataset -from apps.export_app.forms import ExportForm +from apps.export_app.forms import ExportForm, RestoreForm from apps.export_app.resources.accounts import AccountResource from apps.export_app.resources.transactions import ( TransactionResource, TransactionTagResource, TransactionEntityResource, TransactionCategoyResource, + InstallmentPlanResource, + RecurringTransactionResource, ) from apps.export_app.resources.currencies import ( CurrencyResource, ExchangeRateResource, ExchangeRateServiceResource, ) +from apps.export_app.resources.rules import ( + TransactionRuleResource, + TransactionRuleActionResource, + UpdateOrCreateTransactionRuleResource, +) +from apps.export_app.resources.dca import ( + DCAStrategyResource, + DCAEntryResource, +) +from apps.export_app.resources.import_app import ( + ImportProfileResource, +) +from apps.common.decorators.htmx import only_htmx + +logger = logging.getLogger() -# Create your views here. +@login_required +@require_http_methods(["GET"]) def export_index(request): - dataset = TransactionResource().export() - print(dataset.csv) + return render(request, "export_app/pages/index.html") +@only_htmx +@login_required +@require_http_methods(["GET", "POST"]) def export_form(request): + timestamp = timezone.localtime(timezone.now()).strftime("%Y-%m-%dT%H-%M-%S") + if request.method == "POST": form = ExportForm(request.POST) if form.is_valid(): @@ -37,10 +67,18 @@ def export_form(request): export_categories = form.cleaned_data.get("categories", False) export_tags = form.cleaned_data.get("tags", False) export_entities = form.cleaned_data.get("entities", False) + export_installment_plans = form.cleaned_data.get("installment_plans", False) + export_recurring_transactions = form.cleaned_data.get( + "recurring_transactions", False + ) + export_exchange_rates_services = form.cleaned_data.get( "exchange_rates_services", False ) export_exchange_rates = form.cleaned_data.get("exchange_rates", False) + export_rules = form.cleaned_data.get("rules", False) + export_dca = form.cleaned_data.get("dca", False) + export_import_profiles = form.cleaned_data.get("import_profiles", False) exports = [] if export_accounts: @@ -49,15 +87,23 @@ def export_form(request): exports.append((CurrencyResource().export(), "currencies")) if export_transactions: exports.append((TransactionResource().export(), "transactions")) + if export_categories: + exports.append( + (TransactionCategoyResource().export(), "transactions_categories") + ) if export_tags: exports.append((TransactionTagResource().export(), "transactions_tags")) if export_entities: exports.append( (TransactionEntityResource().export(), "transactions_entities") ) - if export_categories: + if export_installment_plans: exports.append( - (TransactionCategoyResource().export(), "transactions_categories") + (InstallmentPlanResource().export(), "installment_plans") + ) + if export_recurring_transactions: + exports.append( + (RecurringTransactionResource().export(), "recurring_transactions") ) if export_exchange_rates_services: exports.append( @@ -65,6 +111,32 @@ def export_form(request): ) if export_exchange_rates: exports.append((ExchangeRateResource().export(), "exchange_rates")) + if export_rules: + exports.append( + (TransactionRuleResource().export(), "transaction_rules") + ) + exports.append( + ( + TransactionRuleActionResource().export(), + "transaction_rules_actions", + ) + ) + exports.append( + ( + UpdateOrCreateTransactionRuleResource().export(), + "transaction_rules_update_or_create", + ) + ) + if export_dca: + exports.append((DCAStrategyResource().export(), "dca_strategies")) + exports.append( + ( + DCAEntryResource().export(), + "dca_entries", + ) + ) + if export_import_profiles: + exports.append((ImportProfileResource().export(), "import_profiles")) if len(exports) >= 2: with zipfile.ZipFile(zip_buffer, "w", zipfile.ZIP_DEFLATED) as zip_file: @@ -72,21 +144,141 @@ def export_form(request): zip_file.writestr(f"{name}.csv", dataset.csv) response = HttpResponse( - zip_buffer.getvalue(), content_type="application/zip" + zip_buffer.getvalue(), + content_type="application/zip", + headers={ + "HX-Trigger": "hide_offcanvas, updated", + "Content-Disposition": f'attachment; filename="{timestamp}_WYGIWYH_export.zip"', + }, ) - response["Content-Disposition"] = f'attachment; filename="export.zip"' return response - else: + elif len(exports) == 1: dataset, name = exports[0] response = HttpResponse( dataset.csv, content_type="text/csv", + headers={ + "HX-Trigger": "hide_offcanvas, updated", + "Content-Disposition": f'attachment; filename="{timestamp}_WYGIWYH_export_{name}.csv"', + }, ) - response["Content-Disposition"] = f'attachment; filename="{name}.csv"' return response + else: + return HttpResponse( + _("You have to select at least one export"), + ) else: form = ExportForm() - return render(request, "export_app/pages/form.html", context={"form": form}) + return render(request, "export_app/fragments/export.html", context={"form": form}) + + +@only_htmx +@login_required +@require_http_methods(["GET", "POST"]) +def import_form(request): + if request.method == "POST": + form = RestoreForm(request.POST, request.FILES) + if form.is_valid(): + try: + process_imports(request, form.cleaned_data) + messages.success(request, _("Data imported successfully")) + return HttpResponse( + status=204, + headers={ + "HX-Trigger": "hide_offcanvas, updated", + }, + ) + except Exception as e: + logger.error("Error importing", exc_info=e) + messages.error( + request, + _("There was an error importing. Check the logs for more details."), + ) + else: + form = RestoreForm() + + response = render(request, "export_app/fragments/restore.html", {"form": form}) + response["HX-Trigger"] = "updated" + return response + + +def process_imports(request, cleaned_data): + # Define import order to handle dependencies + import_order = [ + ("currencies", CurrencyResource), + ( + "currencies", + CurrencyResource, + ), # We do a double pass because exchange_currency may not exist when currency is initially created + ("accounts", AccountResource), + ("transactions_categories", TransactionCategoyResource), + ("transactions_tags", TransactionTagResource), + ("transactions_entities", TransactionEntityResource), + ("automatic_exchange_rates", ExchangeRateServiceResource), + ("exchange_rates", ExchangeRateResource), + ("installment_plans", InstallmentPlanResource), + ("recurring_transactions", RecurringTransactionResource), + ("transactions", TransactionResource), + ("dca_strategies", DCAStrategyResource), + ("dca_entries", DCAEntryResource), + ("import_profiles", ImportProfileResource), + ("transaction_rules", TransactionRuleResource), + ("transaction_rules_actions", TransactionRuleActionResource), + ("transaction_rules_update_or_create", UpdateOrCreateTransactionRuleResource), + ] + + def import_dataset(content, resource_class, field_name): + try: + # Create a new resource instance + resource = resource_class() + + # Create dataset from CSV content + dataset = Dataset() + dataset.load(content, format="csv") + + # Debug logging + logger.debug(f"Importing {field_name}") + logger.debug(f"Headers: {dataset.headers}") + logger.debug(f"First row: {dataset[0] if len(dataset) > 0 else 'No data'}") + + # Perform the import + result = resource.import_data( + dataset, + dry_run=False, + raise_errors=True, + collect_failed_rows=True, + use_transactions=False, + skip_unchanged=True, + ) + + if result.has_errors(): + raise ImportError(f"Failed rows: {result.failed_dataset}") + + return result + + except Exception as e: + logger.error(f"Error importing {field_name}: {str(e)}") + raise ImportError(f"Error importing {field_name}: {str(e)}") + + with transaction.atomic(): + if zip_file := cleaned_data.get("zip_file"): + # Process ZIP file + with zipfile.ZipFile(zip_file) as z: + for filename in z.namelist(): + name = filename.replace(".csv", "") + with z.open(filename) as f: + content = f.read().decode("utf-8") + + for field_name, resource_class in import_order: + if name == field_name: + import_dataset(content, resource_class, field_name) + break + else: + # Process individual files + for field_name, resource_class in import_order: + if csv_file := cleaned_data.get(field_name): + content = csv_file.read().decode("utf-8") + import_dataset(content, resource_class, field_name) diff --git a/app/apps/export_app/widgets/string.py b/app/apps/export_app/widgets/string.py new file mode 100644 index 0000000..7a133aa --- /dev/null +++ b/app/apps/export_app/widgets/string.py @@ -0,0 +1,7 @@ +from import_export import fields + + +class EmptyStringToNoneField(fields.Field): + def clean(self, data, **kwargs): + value = super().clean(data) + return None if value == "" else value diff --git a/app/templates/export_app/fragments/export.html b/app/templates/export_app/fragments/export.html new file mode 100644 index 0000000..241dd2f --- /dev/null +++ b/app/templates/export_app/fragments/export.html @@ -0,0 +1,13 @@ +{% extends "extends/offcanvas.html" %} +{% load crispy_forms_tags %} +{% load i18n %} + +{% block title %}{% translate 'Export' %}{% endblock %} + +{% block body %} +
+
+ {% crispy form %} +
+
+{% endblock %} diff --git a/app/templates/export_app/fragments/restore.html b/app/templates/export_app/fragments/restore.html new file mode 100644 index 0000000..c158a75 --- /dev/null +++ b/app/templates/export_app/fragments/restore.html @@ -0,0 +1,17 @@ +{% extends "extends/offcanvas.html" %} +{% load crispy_forms_tags %} +{% load i18n %} + +{% block title %}{% translate 'Restore' %}{% endblock %} + +{% block body %} +
+
+ {% crispy form %} +
+
+{% endblock %} diff --git a/app/templates/export_app/pages/form.html b/app/templates/export_app/pages/form.html deleted file mode 100644 index 7144e99..0000000 --- a/app/templates/export_app/pages/form.html +++ /dev/null @@ -1,13 +0,0 @@ -{% extends "layouts/base.html" %} -{% load crispy_forms_tags %} -{% load i18n %} - -{% block title %}{% translate 'Import Profiles' %}{% endblock %} - -{% block content %} -
-
- {% crispy form %} -
-
-{% endblock %} diff --git a/app/templates/export_app/pages/index.html b/app/templates/export_app/pages/index.html new file mode 100644 index 0000000..c8bd992 --- /dev/null +++ b/app/templates/export_app/pages/index.html @@ -0,0 +1,29 @@ +{% extends "layouts/base.html" %} +{% load i18n %} + +{% block title %}{% translate 'Export and Restore' %}{% endblock %} + +{% block content %} +
+
+
+ +
+
+ +
+
+
+{% endblock %} diff --git a/app/templates/includes/navbar.html b/app/templates/includes/navbar.html index 53c06c0..1724b55 100644 --- a/app/templates/includes/navbar.html +++ b/app/templates/includes/navbar.html @@ -94,7 +94,7 @@
  • -
  • {% translate 'Import' %} beta
    • +
  • {% translate 'Export and Restore' %}
  • {% translate 'Automatic Exchange Rates' %}