feat: export (WIP)

This commit is contained in:
Herculino Trotta
2025-02-19 13:38:00 -03:00
parent 069f1b450c
commit 201ccea842
15 changed files with 538 additions and 51 deletions

View File

@@ -1,6 +1,6 @@
from crispy_forms.bootstrap import FormActions
from crispy_forms.helper import FormHelper
from crispy_forms.layout import Layout
from crispy_forms.layout import Layout, HTML
from django import forms
from django.utils.translation import gettext_lazy as _
@@ -44,6 +44,18 @@ class ExportForm(forms.Form):
label=_("Entities"),
initial=False,
)
recurring_transactions = forms.BooleanField(
required=False,
widget=forms.CheckboxInput(),
label=_("Recurring Transactions"),
initial=True,
)
installment_plans = forms.BooleanField(
required=False,
widget=forms.CheckboxInput(),
label=_("Installment Planss"),
initial=True,
)
exchange_rates = forms.BooleanField(
required=False,
widget=forms.CheckboxInput(),
@@ -56,6 +68,24 @@ class ExportForm(forms.Form):
label=_("Automatic Exchange Rates"),
initial=False,
)
rules = forms.BooleanField(
required=False,
widget=forms.CheckboxInput(),
label=_("Rules"),
initial=True,
)
dca = forms.BooleanField(
required=False,
widget=forms.CheckboxInput(),
label=_("DCA"),
initial=False,
)
import_profiles = forms.BooleanField(
required=False,
widget=forms.CheckboxInput(),
label=_("Import Profiles"),
initial=True,
)
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
@@ -70,11 +100,90 @@ class ExportForm(forms.Form):
"categories",
"entities",
"tags",
"installment_plans",
"recurring_transactions",
"exchange_rates_services",
"exchange_rates",
"rules",
"dca",
"import_profiles",
FormActions(
NoClassSubmit(
"submit", _("Export"), css_class="btn btn-outline-primary w-100"
),
),
)
class RestoreForm(forms.Form):
zip_file = forms.FileField(
required=False,
help_text=_("Import a ZIP file exported from WYGIWYH"),
label=_("ZIP File"),
)
accounts = forms.FileField(required=False, label=_("Accounts"))
currencies = forms.FileField(required=False, label=_("Currencies"))
transactions_categories = forms.FileField(required=False, label=_("Categories"))
transactions_tags = forms.FileField(required=False, label=_("Tags"))
transactions_entities = forms.FileField(required=False, label=_("Entities"))
transactions = forms.FileField(required=False, label=_("Transactions"))
installment_plans = forms.FileField(required=False, label=_("Installment Plans"))
recurring_transactions = forms.FileField(
required=False, label=_("Recurring Transactions")
)
automatic_exchange_rates = forms.FileField(
required=False, label=_("Automatic Exchange Rates")
)
exchange_rates = forms.FileField(required=False, label=_("Exchange Rates"))
transaction_rules = forms.FileField(required=False, label=_("Transaction rules"))
transaction_rules_actions = forms.FileField(
required=False, label=_("Edit transaction action")
)
transaction_rules_update_or_create = forms.FileField(
required=False, label=_("Update or create transaction actions")
)
dca_strategies = forms.FileField(required=False, label=_("DCA Strategies"))
dca_entries = forms.FileField(required=False, label=_("DCA Entries"))
import_profiles = forms.FileField(required=False, label=_("Import Profiles"))
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.helper = FormHelper()
self.helper.form_tag = False
self.helper.form_method = "post"
self.helper.layout = Layout(
"zip_file",
HTML("<hr />"),
"accounts",
"currencies",
"transactions",
"transactions_categories",
"transactions_entities",
"transactions_tags",
"installment_plans",
"recurring_transactions",
"automatic_exchange_rates",
"exchange_rates",
"transaction_rules",
"transaction_rules_actions",
"transaction_rules_update_or_create",
"dca_strategies",
"dca_entries",
"import_profiles",
FormActions(
NoClassSubmit(
"submit", _("Restore"), css_class="btn btn-outline-primary w-100"
),
),
)
def clean(self):
cleaned_data = super().clean()
if not cleaned_data.get("zip_file") and not any(
cleaned_data.get(field) for field in self.fields if field != "zip_file"
):
raise forms.ValidationError(
_("Please upload either a ZIP file or at least one CSV file")
)
return cleaned_data

View File

@@ -1,24 +1,25 @@
from import_export import fields, resources, widgets
from apps.accounts.models import Account
from apps.accounts.models import Account, AccountGroup
from apps.export_app.widgets.foreign_key import AutoCreateForeignKeyWidget
from apps.currencies.models import Currency
class AccountResource(resources.ModelResource):
group = fields.Field(
attribute="group",
column_name="group",
widget=AutoCreateForeignKeyWidget("accounts.AccountGroup", "name"),
widget=AutoCreateForeignKeyWidget(AccountGroup, "name"),
)
currency = fields.Field(
attribute="currency",
column_name="currency",
widget=widgets.ForeignKeyWidget("currencies.Currency", "name"),
widget=widgets.ForeignKeyWidget(Currency, "name"),
)
exchange_currency = fields.Field(
attribute="exchange_currency",
column_name="exchange_currency",
widget=widgets.ForeignKeyWidget("currencies.Currency", "name"),
widget=widgets.ForeignKeyWidget(Currency, "name"),
)
class Meta:

View File

@@ -1,5 +1,6 @@
from import_export import fields, resources, widgets
from apps.accounts.models import Account
from apps.currencies.models import Currency, ExchangeRate, ExchangeRateService
@@ -7,7 +8,7 @@ class CurrencyResource(resources.ModelResource):
exchange_currency = fields.Field(
attribute="exchange_currency",
column_name="exchange_currency",
widget=widgets.ForeignKeyWidget("currencies.Currency", "name"),
widget=widgets.ForeignKeyWidget(Currency, "name"),
)
class Meta:
@@ -18,12 +19,12 @@ class ExchangeRateResource(resources.ModelResource):
from_currency = fields.Field(
attribute="from_currency",
column_name="from_currency",
widget=widgets.ForeignKeyWidget("currencies.Currency", "name"),
widget=widgets.ForeignKeyWidget(Currency, "name"),
)
to_currency = fields.Field(
attribute="to_currency",
column_name="to_currency",
widget=widgets.ForeignKeyWidget("currencies.Currency", "name"),
widget=widgets.ForeignKeyWidget(Currency, "name"),
)
class Meta:
@@ -34,12 +35,12 @@ class ExchangeRateServiceResource(resources.ModelResource):
target_currencies = fields.Field(
attribute="target_currencies",
column_name="target_currencies",
widget=widgets.ManyToManyWidget("currencies.Currency", field="name"),
widget=widgets.ManyToManyWidget(Currency, field="name"),
)
target_accounts = fields.Field(
attribute="target_accounts",
column_name="target_accounts",
widget=widgets.ForeignKeyWidget("accounts.Account", field="name"),
widget=widgets.ManyToManyWidget(Account, field="name"),
)
class Meta:

View File

@@ -0,0 +1,26 @@
from import_export import fields, resources
from import_export.widgets import ForeignKeyWidget
from apps.dca.models import DCAStrategy, DCAEntry
from apps.currencies.models import Currency
class DCAStrategyResource(resources.ModelResource):
target_currency = fields.Field(
attribute="target_currency",
column_name="target_currency",
widget=ForeignKeyWidget(Currency, "name"),
)
payment_currency = fields.Field(
attribute="payment_currency",
column_name="payment_currency",
widget=ForeignKeyWidget(Currency, "name"),
)
class Meta:
model = DCAStrategy
class DCAEntryResource(resources.ModelResource):
class Meta:
model = DCAEntry

View File

@@ -0,0 +1,8 @@
from import_export import resources
from apps.import_app.models import ImportProfile
class ImportProfileResource(resources.ModelResource):
class Meta:
model = ImportProfile

View File

@@ -0,0 +1,25 @@
from import_export import fields, resources
from import_export.widgets import ForeignKeyWidget
from apps.export_app.widgets.foreign_key import AutoCreateForeignKeyWidget
from apps.export_app.widgets.many_to_many import AutoCreateManyToManyWidget
from apps.rules.models import (
TransactionRule,
TransactionRuleAction,
UpdateOrCreateTransactionRuleAction,
)
class TransactionRuleResource(resources.ModelResource):
class Meta:
model = TransactionRule
class TransactionRuleActionResource(resources.ModelResource):
class Meta:
model = TransactionRuleAction
class UpdateOrCreateTransactionRuleResource(resources.ModelResource):
class Meta:
model = UpdateOrCreateTransactionRuleAction

View File

@@ -1,13 +1,17 @@
from import_export import fields, resources
from import_export.widgets import ForeignKeyWidget
from apps.accounts.models import Account
from apps.export_app.widgets.foreign_key import AutoCreateForeignKeyWidget
from apps.export_app.widgets.many_to_many import AutoCreateManyToManyWidget
from apps.export_app.widgets.string import EmptyStringToNoneField
from apps.transactions.models import (
Transaction,
TransactionCategory,
TransactionTag,
TransactionEntity,
RecurringTransaction,
InstallmentPlan,
)
@@ -15,7 +19,58 @@ class TransactionResource(resources.ModelResource):
account = fields.Field(
attribute="account",
column_name="account",
widget=ForeignKeyWidget("accounts.Account", "name"),
widget=ForeignKeyWidget(Account, "name"),
)
category = fields.Field(
attribute="category",
column_name="category",
widget=AutoCreateForeignKeyWidget(TransactionCategory, "name"),
)
tags = fields.Field(
attribute="tags",
column_name="tags",
widget=AutoCreateManyToManyWidget(TransactionTag, field="name"),
)
entities = fields.Field(
attribute="entities",
column_name="entities",
widget=AutoCreateManyToManyWidget(TransactionEntity, field="name"),
)
internal_id = EmptyStringToNoneField(
column_name="internal_id", attribute="internal_id"
)
class Meta:
model = Transaction
def get_queryset(self):
return Transaction.all_objects.all()
class TransactionTagResource(resources.ModelResource):
class Meta:
model = TransactionTag
class TransactionEntityResource(resources.ModelResource):
class Meta:
model = TransactionEntity
class TransactionCategoyResource(resources.ModelResource):
class Meta:
model = TransactionCategory
class RecurringTransactionResource(resources.ModelResource):
account = fields.Field(
attribute="account",
column_name="account",
widget=ForeignKeyWidget(Account, "name"),
)
category = fields.Field(
@@ -37,19 +92,33 @@ class TransactionResource(resources.ModelResource):
)
class Meta:
model = Transaction
model = RecurringTransaction
class TransactionTagResource(resources.ModelResource):
class InstallmentPlanResource(resources.ModelResource):
account = fields.Field(
attribute="account",
column_name="account",
widget=ForeignKeyWidget(Account, "name"),
)
category = fields.Field(
attribute="category",
column_name="category",
widget=AutoCreateForeignKeyWidget(TransactionCategory, "name"),
)
tags = fields.Field(
attribute="tags",
column_name="tags",
widget=AutoCreateManyToManyWidget(TransactionTag, field="name"),
)
entities = fields.Field(
attribute="entities",
column_name="entities",
widget=AutoCreateManyToManyWidget(TransactionEntity, field="name"),
)
class Meta:
model = TransactionTag
class TransactionEntityResource(resources.ModelResource):
class Meta:
model = TransactionEntity
class TransactionCategoyResource(resources.ModelResource):
class Meta:
model = TransactionCategory
model = InstallmentPlan

View File

@@ -3,5 +3,6 @@ import apps.export_app.views as views
urlpatterns = [
path("export/", views.export_index, name="export_index"),
path("export/export/", views.export_form, name="export_form"),
path("export/form/", views.export_form, name="export_form"),
path("export/restore/", views.import_form, name="restore_form"),
]

View File

@@ -1,31 +1,61 @@
import logging
import zipfile
from io import BytesIO
from io import BytesIO, TextIOWrapper
from django.contrib import messages
from django.contrib.auth.decorators import login_required
from django.db import transaction
from django.http import HttpResponse
from django.shortcuts import render
from django.utils import timezone
from django.utils.translation import gettext_lazy as _
from django.views.decorators.http import require_http_methods
from tablib import Dataset
from apps.export_app.forms import ExportForm
from apps.export_app.forms import ExportForm, RestoreForm
from apps.export_app.resources.accounts import AccountResource
from apps.export_app.resources.transactions import (
TransactionResource,
TransactionTagResource,
TransactionEntityResource,
TransactionCategoyResource,
InstallmentPlanResource,
RecurringTransactionResource,
)
from apps.export_app.resources.currencies import (
CurrencyResource,
ExchangeRateResource,
ExchangeRateServiceResource,
)
from apps.export_app.resources.rules import (
TransactionRuleResource,
TransactionRuleActionResource,
UpdateOrCreateTransactionRuleResource,
)
from apps.export_app.resources.dca import (
DCAStrategyResource,
DCAEntryResource,
)
from apps.export_app.resources.import_app import (
ImportProfileResource,
)
from apps.common.decorators.htmx import only_htmx
logger = logging.getLogger()
# Create your views here.
@login_required
@require_http_methods(["GET"])
def export_index(request):
dataset = TransactionResource().export()
print(dataset.csv)
return render(request, "export_app/pages/index.html")
@only_htmx
@login_required
@require_http_methods(["GET", "POST"])
def export_form(request):
timestamp = timezone.localtime(timezone.now()).strftime("%Y-%m-%dT%H-%M-%S")
if request.method == "POST":
form = ExportForm(request.POST)
if form.is_valid():
@@ -37,10 +67,18 @@ def export_form(request):
export_categories = form.cleaned_data.get("categories", False)
export_tags = form.cleaned_data.get("tags", False)
export_entities = form.cleaned_data.get("entities", False)
export_installment_plans = form.cleaned_data.get("installment_plans", False)
export_recurring_transactions = form.cleaned_data.get(
"recurring_transactions", False
)
export_exchange_rates_services = form.cleaned_data.get(
"exchange_rates_services", False
)
export_exchange_rates = form.cleaned_data.get("exchange_rates", False)
export_rules = form.cleaned_data.get("rules", False)
export_dca = form.cleaned_data.get("dca", False)
export_import_profiles = form.cleaned_data.get("import_profiles", False)
exports = []
if export_accounts:
@@ -49,15 +87,23 @@ def export_form(request):
exports.append((CurrencyResource().export(), "currencies"))
if export_transactions:
exports.append((TransactionResource().export(), "transactions"))
if export_categories:
exports.append(
(TransactionCategoyResource().export(), "transactions_categories")
)
if export_tags:
exports.append((TransactionTagResource().export(), "transactions_tags"))
if export_entities:
exports.append(
(TransactionEntityResource().export(), "transactions_entities")
)
if export_categories:
if export_installment_plans:
exports.append(
(TransactionCategoyResource().export(), "transactions_categories")
(InstallmentPlanResource().export(), "installment_plans")
)
if export_recurring_transactions:
exports.append(
(RecurringTransactionResource().export(), "recurring_transactions")
)
if export_exchange_rates_services:
exports.append(
@@ -65,6 +111,32 @@ def export_form(request):
)
if export_exchange_rates:
exports.append((ExchangeRateResource().export(), "exchange_rates"))
if export_rules:
exports.append(
(TransactionRuleResource().export(), "transaction_rules")
)
exports.append(
(
TransactionRuleActionResource().export(),
"transaction_rules_actions",
)
)
exports.append(
(
UpdateOrCreateTransactionRuleResource().export(),
"transaction_rules_update_or_create",
)
)
if export_dca:
exports.append((DCAStrategyResource().export(), "dca_strategies"))
exports.append(
(
DCAEntryResource().export(),
"dca_entries",
)
)
if export_import_profiles:
exports.append((ImportProfileResource().export(), "import_profiles"))
if len(exports) >= 2:
with zipfile.ZipFile(zip_buffer, "w", zipfile.ZIP_DEFLATED) as zip_file:
@@ -72,21 +144,141 @@ def export_form(request):
zip_file.writestr(f"{name}.csv", dataset.csv)
response = HttpResponse(
zip_buffer.getvalue(), content_type="application/zip"
zip_buffer.getvalue(),
content_type="application/zip",
headers={
"HX-Trigger": "hide_offcanvas, updated",
"Content-Disposition": f'attachment; filename="{timestamp}_WYGIWYH_export.zip"',
},
)
response["Content-Disposition"] = f'attachment; filename="export.zip"'
return response
else:
elif len(exports) == 1:
dataset, name = exports[0]
response = HttpResponse(
dataset.csv,
content_type="text/csv",
headers={
"HX-Trigger": "hide_offcanvas, updated",
"Content-Disposition": f'attachment; filename="{timestamp}_WYGIWYH_export_{name}.csv"',
},
)
response["Content-Disposition"] = f'attachment; filename="{name}.csv"'
return response
else:
return HttpResponse(
_("You have to select at least one export"),
)
else:
form = ExportForm()
return render(request, "export_app/pages/form.html", context={"form": form})
return render(request, "export_app/fragments/export.html", context={"form": form})
@only_htmx
@login_required
@require_http_methods(["GET", "POST"])
def import_form(request):
if request.method == "POST":
form = RestoreForm(request.POST, request.FILES)
if form.is_valid():
try:
process_imports(request, form.cleaned_data)
messages.success(request, _("Data imported successfully"))
return HttpResponse(
status=204,
headers={
"HX-Trigger": "hide_offcanvas, updated",
},
)
except Exception as e:
logger.error("Error importing", exc_info=e)
messages.error(
request,
_("There was an error importing. Check the logs for more details."),
)
else:
form = RestoreForm()
response = render(request, "export_app/fragments/restore.html", {"form": form})
response["HX-Trigger"] = "updated"
return response
def process_imports(request, cleaned_data):
# Define import order to handle dependencies
import_order = [
("currencies", CurrencyResource),
(
"currencies",
CurrencyResource,
), # We do a double pass because exchange_currency may not exist when currency is initially created
("accounts", AccountResource),
("transactions_categories", TransactionCategoyResource),
("transactions_tags", TransactionTagResource),
("transactions_entities", TransactionEntityResource),
("automatic_exchange_rates", ExchangeRateServiceResource),
("exchange_rates", ExchangeRateResource),
("installment_plans", InstallmentPlanResource),
("recurring_transactions", RecurringTransactionResource),
("transactions", TransactionResource),
("dca_strategies", DCAStrategyResource),
("dca_entries", DCAEntryResource),
("import_profiles", ImportProfileResource),
("transaction_rules", TransactionRuleResource),
("transaction_rules_actions", TransactionRuleActionResource),
("transaction_rules_update_or_create", UpdateOrCreateTransactionRuleResource),
]
def import_dataset(content, resource_class, field_name):
try:
# Create a new resource instance
resource = resource_class()
# Create dataset from CSV content
dataset = Dataset()
dataset.load(content, format="csv")
# Debug logging
logger.debug(f"Importing {field_name}")
logger.debug(f"Headers: {dataset.headers}")
logger.debug(f"First row: {dataset[0] if len(dataset) > 0 else 'No data'}")
# Perform the import
result = resource.import_data(
dataset,
dry_run=False,
raise_errors=True,
collect_failed_rows=True,
use_transactions=False,
skip_unchanged=True,
)
if result.has_errors():
raise ImportError(f"Failed rows: {result.failed_dataset}")
return result
except Exception as e:
logger.error(f"Error importing {field_name}: {str(e)}")
raise ImportError(f"Error importing {field_name}: {str(e)}")
with transaction.atomic():
if zip_file := cleaned_data.get("zip_file"):
# Process ZIP file
with zipfile.ZipFile(zip_file) as z:
for filename in z.namelist():
name = filename.replace(".csv", "")
with z.open(filename) as f:
content = f.read().decode("utf-8")
for field_name, resource_class in import_order:
if name == field_name:
import_dataset(content, resource_class, field_name)
break
else:
# Process individual files
for field_name, resource_class in import_order:
if csv_file := cleaned_data.get(field_name):
content = csv_file.read().decode("utf-8")
import_dataset(content, resource_class, field_name)

View File

@@ -0,0 +1,7 @@
from import_export import fields
class EmptyStringToNoneField(fields.Field):
def clean(self, data, **kwargs):
value = super().clean(data)
return None if value == "" else value

View File

@@ -0,0 +1,13 @@
{% extends "extends/offcanvas.html" %}
{% load crispy_forms_tags %}
{% load i18n %}
{% block title %}{% translate 'Export' %}{% endblock %}
{% block body %}
<div class="container p-3">
<form method="post" action="{% url 'export_form' %}" id="export-form" class="show-loading px-1" _="on submit trigger hide_offcanvas" target="_blank">
{% crispy form %}
</form>
</div>
{% endblock %}

View File

@@ -0,0 +1,17 @@
{% extends "extends/offcanvas.html" %}
{% load crispy_forms_tags %}
{% load i18n %}
{% block title %}{% translate 'Restore' %}{% endblock %}
{% block body %}
<div class="container p-3">
<form hx-post="{% url 'restore_form' %}"
hx-target="#generic-offcanvas"
id="restore-form"
enctype="multipart/form-data"
class="show-loading px-1">
{% crispy form %}
</form>
</div>
{% endblock %}

View File

@@ -1,13 +0,0 @@
{% extends "layouts/base.html" %}
{% load crispy_forms_tags %}
{% load i18n %}
{% block title %}{% translate 'Import Profiles' %}{% endblock %}
{% block content %}
<div class="container p-3">
<form method="post" action="{% url 'export_form' %}" id="export-form">
{% crispy form %}
</form>
</div>
{% endblock %}

View File

@@ -0,0 +1,29 @@
{% extends "layouts/base.html" %}
{% load i18n %}
{% block title %}{% translate 'Export and Restore' %}{% endblock %}
{% block content %}
<div class="container">
<div class="row d-flex flex-row align-items-center justify-content-center my-5">
<div class="text-center w-auto mb-3">
<button class="btn btn-outline-success d-flex flex-column align-items-center justify-content-center p-3"
style="width: 100px; height: 100px;"
hx-get="{% url 'export_form' %}"
hx-target="#generic-offcanvas">
<i class="fa-solid fa-download mb-1"></i>
<span>{% trans 'Export' %}</span>
</button>
</div>
<div class="text-center w-auto mb-3">
<button class="btn btn-outline-primary d-flex flex-column align-items-center justify-content-center p-3"
style="width: 100px; height: 100px;"
hx-get="{% url 'restore_form' %}"
hx-target="#generic-offcanvas">
<i class="fa-solid fa-upload mb-1"></i>
<span>{% trans 'Restore' %}</span>
</button>
</div>
</div>
</div>
{% endblock %}

View File

@@ -94,7 +94,7 @@
</ul>
</li>
<li class="nav-item dropdown">
<a class="nav-link dropdown-toggle {% active_link views='tags_index||entities_index||categories_index||accounts_index||account_groups_index||currencies_index||exchange_rates_index||rules_index||import_profiles_index||automatic_exchange_rates_index' %}"
<a class="nav-link dropdown-toggle {% active_link views='tags_index||entities_index||categories_index||accounts_index||account_groups_index||currencies_index||exchange_rates_index||rules_index||import_profiles_index||automatic_exchange_rates_index||export_index' %}"
href="#" role="button"
data-bs-toggle="dropdown"
aria-expanded="false">
@@ -132,6 +132,8 @@
href="{% url 'rules_index' %}">{% translate 'Rules' %}</a></li>
<li><a class="dropdown-item {% active_link views='import_profiles_index' %}"
href="{% url 'import_profiles_index' %}">{% translate 'Import' %} <span class="badge text-bg-primary">beta</span></a></li>
<li><a class="dropdown-item {% active_link views='export_index' %}"
href="{% url 'export_index' %}">{% translate 'Export and Restore' %}</a></li>
<li><a class="dropdown-item {% active_link views='automatic_exchange_rates_index' %}"
href="{% url 'automatic_exchange_rates_index' %}">{% translate 'Automatic Exchange Rates' %}</a></li>
<li>