From 620e7df015e7ff344c50fda2563091b655979981 Mon Sep 17 00:00:00 2001 From: Herculino Trotta Date: Sun, 8 Nov 2026 12:39:16 -0300 Subject: [PATCH 1/6] feat(bank-sync): sync bank transactions from external providers Provider registry modeled after automatic exchange rates: each provider declares its own settings fields, rendered dynamically in the service form. Lunch Flow is the first provider. - Map provider accounts to WYGIWYH accounts with a start date and an optional currency filter - Hourly task syncs due services; manual sync from the list - Deduplicates through Transaction.internal_id, optionally matching transactions already entered by hand - Per account errors and provider reported connection problems Refs #498 --- app/WYGIWYH/settings.py | 1 + app/WYGIWYH/urls.py | 1 + app/apps/bank_sync/__init__.py | 0 app/apps/bank_sync/admin.py | 6 + app/apps/bank_sync/apps.py | 6 + app/apps/bank_sync/forms.py | 126 ++++ app/apps/bank_sync/migrations/0001_initial.py | 61 ++ app/apps/bank_sync/migrations/__init__.py | 0 app/apps/bank_sync/models.py | 102 +++ app/apps/bank_sync/providers/__init__.py | 6 + app/apps/bank_sync/providers/base.py | 49 ++ app/apps/bank_sync/providers/lunchflow.py | 101 +++ app/apps/bank_sync/sync.py | 172 ++++++ app/apps/bank_sync/tasks.py | 17 + app/apps/bank_sync/tests/__init__.py | 0 app/apps/bank_sync/tests/test_bank_sync.py | 583 ++++++++++++++++++ app/apps/bank_sync/urls.py | 21 + app/apps/bank_sync/views.py | 135 ++++ .../bank_sync/fragments/accounts.html | 37 ++ app/templates/bank_sync/fragments/form.html | 10 + app/templates/bank_sync/fragments/list.html | 84 +++ app/templates/bank_sync/pages/index.html | 8 + 22 files changed, 1526 insertions(+) create mode 100644 app/apps/bank_sync/__init__.py create mode 100644 app/apps/bank_sync/admin.py create mode 100644 app/apps/bank_sync/apps.py create mode 100644 app/apps/bank_sync/forms.py create mode 100644 app/apps/bank_sync/migrations/0001_initial.py create mode 100644 app/apps/bank_sync/migrations/__init__.py create mode 100644 app/apps/bank_sync/models.py create mode 100644 app/apps/bank_sync/providers/__init__.py create mode 100644 app/apps/bank_sync/providers/base.py create mode 100644 app/apps/bank_sync/providers/lunchflow.py create mode 100644 app/apps/bank_sync/sync.py create mode 100644 app/apps/bank_sync/tasks.py create mode 100644 app/apps/bank_sync/tests/__init__.py create mode 100644 app/apps/bank_sync/tests/test_bank_sync.py create mode 100644 app/apps/bank_sync/urls.py create mode 100644 app/apps/bank_sync/views.py create mode 100644 app/templates/bank_sync/fragments/accounts.html create mode 100644 app/templates/bank_sync/fragments/form.html create mode 100644 app/templates/bank_sync/fragments/list.html create mode 100644 app/templates/bank_sync/pages/index.html diff --git a/app/WYGIWYH/settings.py b/app/WYGIWYH/settings.py index f7756b73..c87e0f93 100644 --- a/app/WYGIWYH/settings.py +++ b/app/WYGIWYH/settings.py @@ -63,6 +63,7 @@ INSTALLED_APPS = [ "procrastinate.contrib.django", "apps.transactions.apps.TransactionsConfig", "apps.currencies.apps.CurrenciesConfig", + "apps.bank_sync.apps.BankSyncConfig", "apps.accounts.apps.AccountsConfig", "apps.net_worth.apps.NetWorthConfig", "apps.import_app.apps.ImportConfig", diff --git a/app/WYGIWYH/urls.py b/app/WYGIWYH/urls.py index 0a442852..10cd8fc6 100644 --- a/app/WYGIWYH/urls.py +++ b/app/WYGIWYH/urls.py @@ -90,6 +90,7 @@ urlpatterns = [ path("", include("apps.monthly_overview.urls")), path("", include("apps.yearly_overview.urls")), path("", include("apps.currencies.urls")), + path("", include("apps.bank_sync.urls")), path("", include("apps.rules.urls")), path("", include("apps.calendar_view.urls")), path("", include("apps.dca.urls")), diff --git a/app/apps/bank_sync/__init__.py b/app/apps/bank_sync/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/app/apps/bank_sync/admin.py b/app/apps/bank_sync/admin.py new file mode 100644 index 00000000..3ce841dc --- /dev/null +++ b/app/apps/bank_sync/admin.py @@ -0,0 +1,6 @@ +from django.contrib import admin + +from apps.bank_sync.models import BankSyncAccount, BankSyncService + +admin.site.register(BankSyncService) +admin.site.register(BankSyncAccount) diff --git a/app/apps/bank_sync/apps.py b/app/apps/bank_sync/apps.py new file mode 100644 index 00000000..e30d3de1 --- /dev/null +++ b/app/apps/bank_sync/apps.py @@ -0,0 +1,6 @@ +from django.apps import AppConfig + + +class BankSyncConfig(AppConfig): + default_auto_field = "django.db.models.BigAutoField" + name = "apps.bank_sync" diff --git a/app/apps/bank_sync/forms.py b/app/apps/bank_sync/forms.py new file mode 100644 index 00000000..924f0234 --- /dev/null +++ b/app/apps/bank_sync/forms.py @@ -0,0 +1,126 @@ +import copy + +from crispy_forms.bootstrap import FormActions +from crispy_forms.helper import FormHelper +from crispy_forms.layout import Layout +from django import forms +from django.db.models import Q +from django.utils.translation import gettext_lazy as _ + +from apps.accounts.models import Account +from apps.bank_sync.models import BankSyncAccount, BankSyncService +from apps.bank_sync.providers import PROVIDER_MAPPING +from apps.common.widgets.crispy.daisyui import Switch +from apps.common.widgets.crispy.submit import NoClassSubmit +from apps.common.widgets.datepicker import AirDatePickerInput + +SETTING_PREFIX = "setting_" + + +class BankSyncServiceForm(forms.ModelForm): + class Meta: + model = BankSyncService + fields = [ + "name", + "service_type", + "is_active", + "sync_interval", + "trigger_rules", + "match_existing", + ] + + def __init__(self, *args, refresh_url=None, **kwargs): + super().__init__(*args, **kwargs) + + if self.instance.pk: + # Synced ids and links belong to the provider, it can't change + self.fields["service_type"].disabled = True + service_type = self.instance.service_type + else: + # The provider's own settings depend on the selected service type + service_type = self.data.get("service_type") or self.initial.get( + "service_type" + ) + provider = PROVIDER_MAPPING.get(service_type) + self.setting_names = [] + for key, field in (provider.settings_fields if provider else {}).items(): + name = SETTING_PREFIX + key + self.fields[name] = copy.deepcopy(field) + if key in self.instance.settings: + self.initial.setdefault(name, self.instance.settings[key]) + self.setting_names.append(name) + + # Re-renders the form with the new provider's fields + self.fields["service_type"].widget.attrs.update( + { + "hx-get": refresh_url, + "hx-include": "closest form", + "hx-target": "#generic-offcanvas", + "hx-trigger": "change", + } + ) + + self.helper = FormHelper() + self.helper.form_tag = False + self.helper.layout = Layout( + "name", + "service_type", + *self.setting_names, + Switch("is_active"), + Switch("trigger_rules"), + Switch("match_existing"), + "sync_interval", + FormActions( + NoClassSubmit( + "submit", + _("Update") if self.instance.pk else _("Add"), + css_class="btn btn-primary", + ), + ), + ) + + def save(self, commit=True): + self.instance.settings = { + name.removeprefix(SETTING_PREFIX): self.cleaned_data[name] + for name in self.setting_names + } + return super().save(commit) + + +class BankSyncAccountForm(forms.ModelForm): + currencies = forms.MultipleChoiceField( + required=False, + widget=forms.CheckboxSelectMultiple, + label=_("Currencies"), + help_text=_("Only sync transactions in these currencies. Empty syncs all."), + ) + + class Meta: + model = BankSyncAccount + fields = ["account", "start_date", "currencies"] + widgets = {"start_date": AirDatePickerInput(clear_button=False)} + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + # Keeps an account archived after being linked, or the whole formset fails + self.fields["account"].queryset = Account.objects.filter( + Q(is_archived=False) | Q(pk=self.instance.account_id) + ).select_related("currency") + self.fields["account"].empty_label = _("Don't sync") + self.fields["currencies"].choices = [ + (code, code) + for code in sorted( + set(self.instance.available_currencies) | set(self.instance.currencies) + ) + ] + + def clean(self): + cleaned_data = super().clean() + if cleaned_data.get("account") and not cleaned_data.get("start_date"): + self.add_error("start_date", _("Required when an account is set.")) + return cleaned_data + + +BankSyncAccountFormSet = forms.modelformset_factory( + BankSyncAccount, form=BankSyncAccountForm, extra=0 +) diff --git a/app/apps/bank_sync/migrations/0001_initial.py b/app/apps/bank_sync/migrations/0001_initial.py new file mode 100644 index 00000000..5f49ee9e --- /dev/null +++ b/app/apps/bank_sync/migrations/0001_initial.py @@ -0,0 +1,61 @@ +# Generated by Django 5.2.17 on 2026-11-08 15:27 + +import django.core.validators +import django.db.models.deletion +from django.conf import settings +from django.db import migrations, models + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + ('accounts', '0016_account_untracked_by'), + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ] + + operations = [ + migrations.CreateModel( + name='BankSyncService', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('name', models.CharField(max_length=255, verbose_name='Name')), + ('service_type', models.CharField(choices=[('lunchflow', 'Lunch Flow')], max_length=255, verbose_name='Service Type')), + ('is_active', models.BooleanField(default=True, verbose_name='Active')), + ('settings', models.JSONField(blank=True, default=dict, verbose_name='Settings')), + ('sync_interval', models.PositiveIntegerField(default=24, help_text='Most banks only update once a day.', validators=[django.core.validators.MinValueValidator(1), django.core.validators.MaxValueValidator(168)], verbose_name='Sync every X hours')), + ('trigger_rules', models.BooleanField(default=True, help_text='Run your transaction rules on synced transactions.', verbose_name='Trigger transaction rules')), + ('match_existing', models.BooleanField(default=False, help_text='Instead of creating a synced transaction, link it to one you already added on the same account, with the same type and amount, dated up to 3 days apart.', verbose_name='Match existing transactions')), + ('last_sync', models.DateTimeField(blank=True, null=True, verbose_name='Last Successful Sync')), + ('failure_count', models.PositiveIntegerField(default=0)), + ('last_error', models.TextField(blank=True, default='')), + ('owner', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='%(class)s_owned', to=settings.AUTH_USER_MODEL)), + ], + options={ + 'verbose_name': 'Bank Sync Service', + 'verbose_name_plural': 'Bank Sync Services', + 'ordering': ['name'], + }, + ), + migrations.CreateModel( + name='BankSyncAccount', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('external_id', models.CharField(max_length=255)), + ('external_name', models.CharField(blank=True, max_length=255)), + ('available_currencies', models.JSONField(blank=True, default=list)), + ('currencies', models.JSONField(blank=True, default=list, help_text='Only sync transactions in these currencies. Empty syncs all.', verbose_name='Currencies')), + ('remote_error', models.TextField(blank=True, default='')), + ('last_error', models.TextField(blank=True, default='')), + ('start_date', models.DateField(blank=True, help_text='Transactions before this date are never imported.', null=True, verbose_name='Sync from')), + ('last_synced_date', models.DateField(blank=True, null=True)), + ('account', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='bank_sync_accounts', to='accounts.account', verbose_name='Account')), + ('service', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='accounts', to='bank_sync.banksyncservice')), + ], + options={ + 'ordering': ['external_name'], + 'unique_together': {('service', 'external_id')}, + }, + ), + ] diff --git a/app/apps/bank_sync/migrations/__init__.py b/app/apps/bank_sync/migrations/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/app/apps/bank_sync/models.py b/app/apps/bank_sync/models.py new file mode 100644 index 00000000..079e4832 --- /dev/null +++ b/app/apps/bank_sync/models.py @@ -0,0 +1,102 @@ +from django.core.validators import MaxValueValidator, MinValueValidator +from django.db import models +from django.utils.translation import gettext_lazy as _ + +from apps.common.models import OwnedObject, OwnedObjectManager + + +class BankSyncService(OwnedObject): + class ServiceType(models.TextChoices): + LUNCHFLOW = "lunchflow", "Lunch Flow" + + name = models.CharField(max_length=255, verbose_name=_("Name")) + service_type = models.CharField( + max_length=255, choices=ServiceType.choices, verbose_name=_("Service Type") + ) + is_active = models.BooleanField(default=True, verbose_name=_("Active")) + # Provider specific, keyed by the provider's settings_fields + settings = models.JSONField(default=dict, blank=True, verbose_name=_("Settings")) + sync_interval = models.PositiveIntegerField( + default=24, + validators=[MinValueValidator(1), MaxValueValidator(168)], + verbose_name=_("Sync every X hours"), + help_text=_("Most banks only update once a day."), + ) + trigger_rules = models.BooleanField( + default=True, + verbose_name=_("Trigger transaction rules"), + help_text=_("Run your transaction rules on synced transactions."), + ) + match_existing = models.BooleanField( + default=False, + verbose_name=_("Match existing transactions"), + help_text=_( + "Instead of creating a synced transaction, link it to one you already " + "added on the same account, with the same type and amount, dated up to " + "3 days apart." + ), + ) + last_sync = models.DateTimeField( + null=True, blank=True, verbose_name=_("Last Successful Sync") + ) + failure_count = models.PositiveIntegerField(default=0) + last_error = models.TextField(blank=True, default="") + + objects = OwnedObjectManager() + + class Meta: + verbose_name = _("Bank Sync Service") + verbose_name_plural = _("Bank Sync Services") + ordering = ["name"] + + def __str__(self): + return self.name + + def get_provider(self): + from apps.bank_sync.providers import PROVIDER_MAPPING + + return PROVIDER_MAPPING[self.service_type](self.settings) + + +class BankSyncAccount(models.Model): + """An account on the provider, optionally linked to a WYGIWYH account.""" + + service = models.ForeignKey( + BankSyncService, on_delete=models.CASCADE, related_name="accounts" + ) + external_id = models.CharField(max_length=255) + external_name = models.CharField(max_length=255, blank=True) + # Currencies seen on the provider, offered as choices for `currencies` + available_currencies = models.JSONField(default=list, blank=True) + currencies = models.JSONField( + default=list, + blank=True, + verbose_name=_("Currencies"), + help_text=_("Only sync transactions in these currencies. Empty syncs all."), + ) + # Provider side problem, like an expired connection + remote_error = models.TextField(blank=True, default="") + last_error = models.TextField(blank=True, default="") + + account = models.ForeignKey( + "accounts.Account", + on_delete=models.SET_NULL, + null=True, + blank=True, + related_name="bank_sync_accounts", + verbose_name=_("Account"), + ) + start_date = models.DateField( + null=True, + blank=True, + verbose_name=_("Sync from"), + help_text=_("Transactions before this date are never imported."), + ) + last_synced_date = models.DateField(null=True, blank=True) + + class Meta: + unique_together = ("service", "external_id") + ordering = ["external_name"] + + def __str__(self): + return self.external_name or self.external_id diff --git a/app/apps/bank_sync/providers/__init__.py b/app/apps/bank_sync/providers/__init__.py new file mode 100644 index 00000000..1edb212d --- /dev/null +++ b/app/apps/bank_sync/providers/__init__.py @@ -0,0 +1,6 @@ +from apps.bank_sync.providers.lunchflow import LunchFlowProvider + +# Keys must match BankSyncService.ServiceType +PROVIDER_MAPPING = { + "lunchflow": LunchFlowProvider, +} diff --git a/app/apps/bank_sync/providers/base.py b/app/apps/bank_sync/providers/base.py new file mode 100644 index 00000000..da1921c9 --- /dev/null +++ b/app/apps/bank_sync/providers/base.py @@ -0,0 +1,49 @@ +from abc import ABC, abstractmethod +from dataclasses import dataclass +from datetime import date +from decimal import Decimal + + +@dataclass(frozen=True) +class RemoteAccount: + id: str + name: str + # Currency codes as the provider spells them + currencies: tuple = () + # Why the account can't be synced right now (expired connection...), empty if fine + error: str = "" + + +@dataclass(frozen=True) +class RemoteTransaction: + id: str + date: date + # Signed: negative is money leaving the account + amount: Decimal + description: str = "" + notes: str = "" + currency: str = "" + + +class BankSyncProvider(ABC): + """A bank data aggregator. + + ``settings_fields`` declares the provider's own configuration as Django form + fields. The service form renders them when the provider is selected and the + values end up in ``BankSyncService.settings`` under the same keys. + """ + + settings_fields: dict = {} + + def __init__(self, settings: dict): + self.settings = settings or {} + + @abstractmethod + def list_accounts(self) -> list[RemoteAccount]: + """Every account the credentials can see.""" + + @abstractmethod + def get_transactions( + self, account_id: str, start_date: date + ) -> list[RemoteTransaction]: + """Posted transactions of ``account_id`` dated ``start_date`` or later.""" diff --git a/app/apps/bank_sync/providers/lunchflow.py b/app/apps/bank_sync/providers/lunchflow.py new file mode 100644 index 00000000..3d7c7c31 --- /dev/null +++ b/app/apps/bank_sync/providers/lunchflow.py @@ -0,0 +1,101 @@ +import os +from datetime import date +from decimal import Decimal + +import requests +from django import forms +from django.utils.translation import gettext as _ +from django.utils.translation import gettext_lazy + +from apps.bank_sync.providers.base import ( + BankSyncProvider, + RemoteAccount, + RemoteTransaction, +) + +DEFAULT_BASE_URL = "https://www.lunchflow.app/api/v1" + + +class LunchFlowProvider(BankSyncProvider): + """https://www.lunchflow.app/docs/api/personal-api-overview""" + + settings_fields = { + "api_key": forms.CharField( + label=gettext_lazy("API Key"), + # Not a PasswordInput: browsers autofill those with saved logins + widget=forms.TextInput(attrs={"autocomplete": "off"}), + help_text=gettext_lazy( + "Create a REST API destination on Lunch Flow and give it access to " + "the accounts you want to sync." + ), + ), + } + + def __init__(self, settings: dict): + super().__init__(settings) + # Server side only, users must not point the server at arbitrary hosts + self.base_url = os.getenv("LUNCHFLOW_API_URL", DEFAULT_BASE_URL).rstrip("/") + self.session = requests.Session() + self.session.headers["x-api-key"] = self.settings.get("api_key", "") + + def _get(self, path: str, **params) -> dict: + response = self.session.get(f"{self.base_url}{path}", params=params, timeout=30) + response.raise_for_status() + return response.json() + + def _balance_currency(self, account_id) -> str: + # The account list has no currency, the balance does + try: + return self._get(f"/accounts/{account_id}/balance")["balance"]["currency"] + except (requests.RequestException, ValueError, KeyError, TypeError): + return "" + + def list_accounts(self) -> list[RemoteAccount]: + accounts = [] + for account in self._get("/accounts")["accounts"]: + currency = account.get("currency") or self._balance_currency(account["id"]) + status = account.get("status") or "ACTIVE" + accounts.append( + RemoteAccount( + id=str(account["id"]), + name=" - ".join( + filter(None, [account.get("institution_name"), account["name"]]) + ), + currencies=(currency.upper(),) if currency else (), + error=( + "" + if status == "ACTIVE" + else _( + "Lunch Flow reports this account as %(status)s, " + "check the connection on Lunch Flow." + ) + % {"status": status} + ), + ) + ) + return accounts + + def get_transactions( + self, account_id: str, start_date: date + ) -> list[RemoteTransaction]: + data = self._get( + f"/accounts/{account_id}/transactions", **{"from": start_date.isoformat()} + ) + transactions = [] + for tx in data["transactions"]: + # Pending ones are excluded by default, but they change ids when posted + if tx.get("isPending"): + continue + merchant = (tx.get("merchant") or "").strip() + description = (tx.get("description") or "").strip() + transactions.append( + RemoteTransaction( + id=str(tx["id"]), + date=date.fromisoformat(tx["date"][:10]), + amount=Decimal(str(tx["amount"])), + description=merchant or description, + notes=description if merchant and description != merchant else "", + currency=(tx.get("currency") or "").upper(), + ) + ) + return transactions diff --git a/app/apps/bank_sync/sync.py b/app/apps/bank_sync/sync.py new file mode 100644 index 00000000..f3e95ed3 --- /dev/null +++ b/app/apps/bank_sync/sync.py @@ -0,0 +1,172 @@ +import logging +from datetime import timedelta + +from django.utils import timezone + +from apps.bank_sync.models import BankSyncAccount, BankSyncService +from apps.common.functions.decimals import truncate_decimal +from apps.common.middleware.thread_local import delete_current_user, write_current_user +from apps.rules.signals import transaction_created +from apps.transactions.models import Transaction + +logger = logging.getLogger(__name__) + +# Banks post late and back-date transactions, so every sync looks this far back +OVERLAP_DAYS = 7 +# How far apart a synced transaction and the one it matches can be dated +MATCH_DAYS = 3 + + +def refresh_remote_accounts(service: BankSyncService, provider=None) -> None: + """Mirror the provider's account list, keeping existing links. + + Accounts the provider no longer returns are dropped: their ids are gone for good. + """ + provider = provider or service.get_provider() + remote_accounts = {remote.id: remote for remote in provider.list_accounts()} + service.accounts.exclude(external_id__in=remote_accounts).delete() + + for remote in remote_accounts.values(): + link, _ = BankSyncAccount.objects.get_or_create( + service=service, external_id=remote.id + ) + link.external_name = remote.name + link.remote_error = remote.error + link.available_currencies = sorted( + set(link.available_currencies) | set(remote.currencies) + ) + link.save() + + +def _match_existing(link: BankSyncAccount, internal_id, type_, amount, date) -> bool: + """Tag a transaction the user already added instead of creating a new one.""" + candidates = Transaction.userless_all_objects.filter( + account=link.account, + type=type_, + amount=truncate_decimal(amount, link.account.currency.decimal_places), + date__range=(date - timedelta(days=MATCH_DAYS), date + timedelta(days=MATCH_DAYS)), + internal_id__isnull=True, + deleted=False, + ) + best = min(candidates, key=lambda t: abs((t.date - date).days), default=None) + if best is None: + return False + # Not a user edit, so no transaction_updated (it would run "on update" rules) + Transaction.userless_all_objects.filter(pk=best.pk).update( + emit_signal=False, internal_id=internal_id + ) + return True + + +def _sync_account(service: BankSyncService, link: BankSyncAccount, provider) -> int: + start = link.start_date + if link.last_synced_date: + start = max(start, link.last_synced_date - timedelta(days=OVERLAP_DAYS)) + + created = 0 + seen_currencies = set() + for remote in provider.get_transactions(link.external_id, start): + if remote.date < link.start_date: + continue + if remote.currency: + seen_currencies.add(remote.currency) + if link.currencies and remote.currency not in link.currencies: + continue + + internal_id = f"{service.service_type}:{link.external_id}:{remote.id}" + # userless_all_objects: also sees soft deleted ones, so deleting a synced + # transaction doesn't bring it back on the next run + if Transaction.userless_all_objects.filter(internal_id=internal_id).exists(): + continue + + type_ = ( + Transaction.Type.INCOME if remote.amount > 0 else Transaction.Type.EXPENSE + ) + amount = abs(remote.amount) + if service.match_existing and _match_existing( + link, internal_id, type_, amount, remote.date + ): + continue + + transaction = Transaction.objects.create( + account=link.account, + owner=service.owner, + type=type_, + amount=amount, + date=remote.date, + description=remote.description[:500], + notes=remote.notes, + is_paid=True, + internal_id=internal_id, + ) + created += 1 + if service.trigger_rules: + transaction_created.send(sender=transaction) + + link.last_synced_date = timezone.localdate() + link.available_currencies = sorted(set(link.available_currencies) | seen_currencies) + link.save(update_fields=["last_synced_date", "available_currencies"]) + return created + + +def sync_service(service: BankSyncService) -> int: + """Import new transactions for every linked account. Returns how many. + + A failing account doesn't stop the others. Any failure keeps last_sync as is, + so the service is retried on the next run. + """ + created = 0 + errors = [] + # Rules and the owned-object managers read the user from the thread + write_current_user(service.owner) + try: + provider = service.get_provider() + refresh_remote_accounts(service, provider) + links = service.accounts.filter( + account__isnull=False, start_date__isnull=False + ).select_related("account__currency") + for link in links: + error = link.remote_error + if not error: + try: + created += _sync_account(service, link, provider) + except Exception as e: + logger.exception(f"Bank sync failed for {service.name} / {link}") + error = str(e) + link.last_error = error[:1000] + link.save(update_fields=["last_error"]) + if error: + errors.append(f"{link}: {error}") + except Exception as e: + logger.exception(f"Bank sync failed for {service.name}") + errors.append(str(e)) + finally: + delete_current_user() + + if errors: + service.failure_count += 1 + service.last_error = "\n".join(errors)[:1000] + service.save(update_fields=["failure_count", "last_error"]) + else: + service.last_sync = timezone.now() + service.failure_count = 0 + service.last_error = "" + service.save(update_fields=["last_sync", "failure_count", "last_error"]) + logger.info(f"Bank sync for {service.name} created {created} transactions") + return created + + +def is_due(service: BankSyncService) -> bool: + if service.last_sync is None: + return True + # Hour granularity, the scheduler runs on the hour + elapsed = timezone.now().replace(minute=0, second=0, microsecond=0) - ( + service.last_sync.replace(minute=0, second=0, microsecond=0) + ) + return elapsed >= timedelta(hours=service.sync_interval) + + +def sync_due_services(force: bool = False) -> None: + for service in BankSyncService.objects.filter(is_active=True): + if force or is_due(service): + sync_service(service) diff --git a/app/apps/bank_sync/tasks.py b/app/apps/bank_sync/tasks.py new file mode 100644 index 00000000..55a363d8 --- /dev/null +++ b/app/apps/bank_sync/tasks.py @@ -0,0 +1,17 @@ +from procrastinate.contrib.django import app + +from apps.bank_sync.models import BankSyncService +from apps.bank_sync.sync import sync_due_services, sync_service + + +@app.periodic(cron="0 * * * *") +@app.task(lock="bank_sync", name="automatic_bank_sync") +def automatic_bank_sync(timestamp=None): + sync_due_services() + + +@app.task(lock="bank_sync", name="manual_bank_sync") +def manual_bank_sync(service_id: int): + service = BankSyncService.objects.filter(id=service_id).first() + if service: + sync_service(service) diff --git a/app/apps/bank_sync/tests/__init__.py b/app/apps/bank_sync/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/app/apps/bank_sync/tests/test_bank_sync.py b/app/apps/bank_sync/tests/test_bank_sync.py new file mode 100644 index 00000000..346bef10 --- /dev/null +++ b/app/apps/bank_sync/tests/test_bank_sync.py @@ -0,0 +1,583 @@ +import io +import os +import zipfile +from datetime import date, timedelta +from decimal import Decimal +from unittest.mock import MagicMock, patch + +from django.contrib.auth import get_user_model +from django.core.files.uploadedfile import SimpleUploadedFile +from django.test import TestCase +from django.urls import reverse +from django.utils import timezone + +from apps.accounts.models import Account +from apps.bank_sync.forms import BankSyncAccountForm, BankSyncServiceForm +from apps.bank_sync.models import BankSyncAccount, BankSyncService +from apps.bank_sync.providers.base import RemoteAccount, RemoteTransaction +from apps.bank_sync.providers.lunchflow import LunchFlowProvider +from apps.bank_sync.sync import is_due, refresh_remote_accounts, sync_service +from apps.common.middleware.thread_local import delete_current_user, write_current_user +from apps.currencies.models import Currency +from apps.transactions.models import Transaction + + +def _response(payload): + response = MagicMock() + response.json.return_value = payload + return response + + +class LunchFlowProviderTests(TestCase): + def setUp(self): + # Independent of whatever the test machine has set + self.enterContext(patch.dict("os.environ")) + os.environ.pop("LUNCHFLOW_API_URL", None) + self.provider = LunchFlowProvider({"api_key": "key"}) + + def test_sends_api_key_and_uses_default_url(self): + self.assertEqual(self.provider.session.headers["x-api-key"], "key") + self.assertEqual(self.provider.base_url, "https://www.lunchflow.app/api/v1") + + def test_url_comes_from_the_environment_not_settings(self): + with patch.dict("os.environ", {"LUNCHFLOW_API_URL": "https://x.test/v1/"}): + provider = LunchFlowProvider({"api_key": "k", "base_url": "http://db:5432"}) + self.assertEqual(provider.base_url, "https://x.test/v1") + + def test_list_accounts(self): + def get(url, **kwargs): + if url.endswith("/accounts"): + return _response( + { + "accounts": [ + {"id": 7, "name": "Checking", "institution_name": "Bank", "status": "ACTIVE"}, + {"id": 8, "name": "Card", "institution_name": None, "status": "EXPIRED"}, + ] + } + ) + if url.endswith("/7/balance"): + return _response({"balance": {"amount": 1, "currency": "eur"}}) + raise __import__("requests").HTTPError("expired") + + with patch.object(self.provider.session, "get", side_effect=get): + accounts = self.provider.list_accounts() + + self.assertEqual(accounts[0], RemoteAccount("7", "Bank - Checking", ("EUR",), "")) + self.assertEqual(accounts[1].name, "Card") + self.assertEqual(accounts[1].currencies, ()) + self.assertIn("EXPIRED", accounts[1].error) + + def test_get_transactions(self): + payload = { + "transactions": [ + {"id": "a", "amount": -12.5, "currency": "EUR", "date": "2026-10-01", "merchant": "Coffee", "description": "CARD 123 COFFEE", "isPending": False}, + {"id": "b", "amount": 1000, "currency": "usd", "date": "2026-10-02", "merchant": None, "description": "Salary"}, + {"id": "c", "amount": -1, "date": "2026-10-03", "merchant": "Pending", "isPending": True}, + {"id": "d", "amount": -3, "date": "2026-10-04", "merchant": "Same", "description": "Same"}, + ] + } + with patch.object( + self.provider.session, "get", return_value=_response(payload) + ) as mock_get: + transactions = self.provider.get_transactions("7", date(2026, 10, 1)) + + mock_get.assert_called_once_with( + "https://www.lunchflow.app/api/v1/accounts/7/transactions", + params={"from": "2026-10-01"}, + timeout=30, + ) + self.assertEqual( + transactions, + [ + RemoteTransaction("a", date(2026, 10, 1), Decimal("-12.5"), "Coffee", "CARD 123 COFFEE", "EUR"), + RemoteTransaction("b", date(2026, 10, 2), Decimal("1000"), "Salary", "", "USD"), + RemoteTransaction("d", date(2026, 10, 4), Decimal("-3"), "Same", "", ""), + ], + ) + + +class FakeProvider: + def __init__(self, transactions=(), accounts=()): + self.transactions = list(transactions) + self.accounts = list(accounts) + self.calls = [] + + def list_accounts(self): + return self.accounts + + def get_transactions(self, account_id, start_date): + self.calls.append((account_id, start_date)) + return [t for t in self.transactions if t.date >= start_date] + + +class SyncTests(TestCase): + def setUp(self): + self.user = get_user_model().objects.create_user( + email="sync@example.com", password="x" + ) + self.eur = Currency.objects.create(code="EUR", name="Euro", decimal_places=2) + self.account = Account.objects.create( + name="Checking", currency=self.eur, owner=self.user + ) + self.service = BankSyncService.objects.create( + name="LF", service_type="lunchflow", owner=self.user + ) + self.link = BankSyncAccount.objects.create( + service=self.service, + external_id="7", + account=self.account, + start_date=date(2026, 10, 1), + ) + self.provider = FakeProvider( + [ + RemoteTransaction("old", date(2026, 9, 30), Decimal("-1"), "Before start"), + RemoteTransaction("a", date(2026, 10, 1), Decimal("-12.555"), "Coffee", "note", "EUR"), + RemoteTransaction("b", date(2026, 10, 2), Decimal("1000"), "Salary", "", "EUR"), + ], + accounts=[RemoteAccount("7", "Bank - Checking", ("EUR",))], + ) + + def _sync(self): + with patch.object(BankSyncService, "get_provider", return_value=self.provider): + created = sync_service(self.service) + self.service.refresh_from_db() + self.link.refresh_from_db() + return created + + def test_creates_transactions(self): + self.assertEqual(self._sync(), 2) + + coffee = Transaction.userless_all_objects.get(internal_id="lunchflow:7:a") + self.assertEqual(coffee.type, Transaction.Type.EXPENSE) + self.assertEqual(coffee.amount, Decimal("12.55")) # truncated to currency + self.assertEqual(coffee.description, "Coffee") + self.assertEqual(coffee.notes, "note") + self.assertEqual(coffee.owner, self.user) + self.assertEqual(coffee.account, self.account) + self.assertTrue(coffee.is_paid) + + salary = Transaction.userless_all_objects.get(internal_id="lunchflow:7:b") + self.assertEqual(salary.type, Transaction.Type.INCOME) + self.assertEqual(salary.amount, Decimal("1000")) + + self.assertIsNotNone(self.service.last_sync) + self.assertEqual(self.link.last_synced_date, timezone.localdate()) + self.assertEqual(self.link.external_name, "Bank - Checking") + + def test_second_run_does_not_duplicate(self): + self._sync() + self.assertEqual(self._sync(), 0) + self.assertEqual(Transaction.userless_all_objects.count(), 2) + + def test_deleted_transactions_are_not_reimported(self): + self._sync() + Transaction.userless_all_objects.filter(internal_id="lunchflow:7:a").update( + deleted=True + ) + self.assertEqual(self._sync(), 0) + + def test_later_runs_only_look_back_the_overlap(self): + self.link.last_synced_date = date(2026, 11, 1) + self.link.save() + self._sync() + self.assertEqual(self.provider.calls, [("7", date(2026, 10, 25))]) + + def test_unlinked_accounts_are_skipped(self): + self.link.account = None + self.link.save() + self.assertEqual(self._sync(), 0) + self.assertEqual(self.provider.calls, []) + + def test_currency_filter(self): + self.provider.transactions.append( + RemoteTransaction("usd", date(2026, 10, 3), Decimal("-5"), "Abroad", "", "USD") + ) + self.link.currencies = ["USD"] + self.link.save() + self.assertEqual(self._sync(), 1) + self.assertTrue( + Transaction.userless_all_objects.filter(internal_id="lunchflow:7:usd").exists() + ) + # Every currency seen becomes a choice, even filtered out ones + self.assertEqual(self.link.available_currencies, ["EUR", "USD"]) + + def test_no_currency_filter_syncs_all(self): + self.provider.transactions.append( + RemoteTransaction("usd", date(2026, 10, 3), Decimal("-5"), "Abroad", "", "USD") + ) + self.assertEqual(self._sync(), 3) + + def test_match_existing_is_off_by_default(self): + Transaction.objects.create( + account=self.account, owner=self.user, type=Transaction.Type.INCOME, + amount=Decimal("1000"), date=date(2026, 10, 2), description="Mine", + ) + self.assertEqual(self._sync(), 2) + + @patch("apps.rules.signals.check_for_transaction_rules.defer") + def test_matching_does_not_trigger_rules(self, defer): + self.service.match_existing = True + self.service.save() + self.provider.transactions = self.provider.transactions[2:] # salary only + Transaction.objects.create( + account=self.account, owner=self.user, type=Transaction.Type.INCOME, + amount=Decimal("1000"), date=date(2026, 10, 2), description="Mine", + ) + defer.reset_mock() + self.assertEqual(self._sync(), 0) + defer.assert_not_called() + + def test_match_existing(self): + self.service.match_existing = True + self.service.save() + far = Transaction.objects.create( + account=self.account, owner=self.user, type=Transaction.Type.INCOME, + amount=Decimal("1000"), date=date(2026, 9, 28), description="Too far", + ) + close = Transaction.objects.create( + account=self.account, owner=self.user, type=Transaction.Type.INCOME, + amount=Decimal("1000"), date=date(2026, 10, 3), description="Mine", + ) + expense = Transaction.objects.create( + account=self.account, owner=self.user, type=Transaction.Type.EXPENSE, + amount=Decimal("12.55"), date=date(2026, 10, 1), description="Coffee by hand", + ) + + self.assertEqual(self._sync(), 0) + close.refresh_from_db() + expense.refresh_from_db() + far.refresh_from_db() + self.assertEqual(close.internal_id, "lunchflow:7:b") + self.assertEqual(close.description, "Mine") # left as the user wrote it + self.assertEqual(expense.internal_id, "lunchflow:7:a") + self.assertIsNone(far.internal_id) + # Matched once, never again + self.assertEqual(self._sync(), 0) + + def test_match_existing_takes_one_per_synced_transaction(self): + self.service.match_existing = True + self.service.save() + self.provider.transactions = [ + RemoteTransaction("x1", date(2026, 10, 5), Decimal("-3"), "Coffee", "", "EUR"), + RemoteTransaction("x2", date(2026, 10, 5), Decimal("-3"), "Coffee", "", "EUR"), + ] + Transaction.objects.create( + account=self.account, owner=self.user, type=Transaction.Type.EXPENSE, + amount=Decimal("3"), date=date(2026, 10, 5), description="One coffee", + ) + self.assertEqual(self._sync(), 1) + self.assertEqual(Transaction.userless_all_objects.count(), 2) + + def test_failing_account_does_not_stop_the_others(self): + other = Account.objects.create(name="Other", currency=self.eur, owner=self.user) + other_link = BankSyncAccount.objects.create( + service=self.service, external_id="8", account=other, start_date=date(2026, 10, 1) + ) + self.provider.accounts.append(RemoteAccount("8", "Broken")) + get_transactions = self.provider.get_transactions + + def flaky(account_id, start_date): + if account_id == "8": + raise Exception("boom") + return get_transactions(account_id, start_date) + + self.provider.get_transactions = flaky + self.assertEqual(self._sync(), 2) + other_link.refresh_from_db() + self.assertEqual(other_link.last_error, "boom") + self.assertEqual(self.link.last_error, "") + self.assertEqual(self.service.failure_count, 1) + self.assertIn("boom", self.service.last_error) + self.assertIsNone(self.service.last_sync) # retried next run + + self.provider.get_transactions = get_transactions + self._sync() + other_link.refresh_from_db() + self.assertEqual(other_link.last_error, "") + self.assertEqual(self.service.failure_count, 0) + self.assertEqual(self.service.last_error, "") + self.assertIsNotNone(self.service.last_sync) + + def test_provider_reported_problem_skips_the_account(self): + self.provider.accounts = [RemoteAccount("7", "Bank", (), "Connection expired")] + self.assertEqual(self._sync(), 0) + self.assertEqual(self.provider.calls, []) + self.assertEqual(self.link.remote_error, "Connection expired") + self.assertEqual(self.link.last_error, "Connection expired") + self.assertEqual(self.service.failure_count, 1) + + def test_listing_failure_is_recorded(self): + self.provider.list_accounts = MagicMock(side_effect=Exception("401")) + self.assertEqual(self._sync(), 0) + self.assertEqual(self.service.failure_count, 1) + self.assertEqual(self.service.last_error, "401") + self.assertTrue(BankSyncAccount.objects.filter(pk=self.link.pk).exists()) + + @patch("apps.rules.signals.check_for_transaction_rules.defer") + def test_rules_are_triggered(self, defer): + self._sync() + self.assertEqual(defer.call_count, 2) + self.assertEqual(defer.call_args.kwargs["user_id"], self.user.id) + + @patch("apps.rules.signals.check_for_transaction_rules.defer") + def test_rules_can_be_disabled(self, defer): + self.service.trigger_rules = False + self.service.save() + self._sync() + defer.assert_not_called() + + def test_refresh_remote_accounts(self): + self.link.available_currencies = ["GBP"] + self.link.save() + gone = BankSyncAccount.objects.create(service=self.service, external_id="gone") + self.provider.accounts = [ + RemoteAccount("7", "Renamed", ("EUR",)), + RemoteAccount("9", "New", ("USD",)), + ] + refresh_remote_accounts(self.service, self.provider) + + self.link.refresh_from_db() + self.assertEqual(self.link.external_name, "Renamed") + self.assertEqual(self.link.account, self.account) + self.assertEqual(self.link.available_currencies, ["EUR", "GBP"]) + self.assertTrue(self.service.accounts.filter(external_id="9", account=None).exists()) + self.assertFalse(BankSyncAccount.objects.filter(pk=gone.pk).exists()) + + def test_is_due(self): + self.service.sync_interval = 24 + self.assertTrue(is_due(self.service)) + self.service.last_sync = timezone.now() - timedelta(hours=23) + self.assertFalse(is_due(self.service)) + self.service.last_sync = timezone.now() - timedelta(hours=25) + self.assertTrue(is_due(self.service)) + + +class FormTests(TestCase): + def setUp(self): + # Account choices are scoped to the logged in user + self.user = get_user_model().objects.create_user(email="f@example.com", password="x") + write_current_user(self.user) + self.addCleanup(delete_current_user) + + def test_provider_settings_fields_are_added(self): + form = BankSyncServiceForm(initial={"service_type": "lunchflow"}) + self.assertIn("setting_api_key", form.fields) + self.assertNotIn("setting_base_url", form.fields) + self.assertNotIn("setting_api_key", BankSyncServiceForm().fields) + + def test_settings_are_saved_and_loaded(self): + form = BankSyncServiceForm( + { + "name": "LF", + "service_type": "lunchflow", + "is_active": "on", + "sync_interval": "24", + "setting_api_key": "secret", + } + ) + self.assertTrue(form.is_valid(), form.errors) + service = form.save() + self.assertEqual(service.settings, {"api_key": "secret"}) + + edit = BankSyncServiceForm(instance=service) + self.assertEqual(edit.initial["setting_api_key"], "secret") + + def test_service_type_is_locked_on_edit(self): + service = BankSyncService.objects.create( + name="LF", service_type="lunchflow", settings={"api_key": "k"} + ) + form = BankSyncServiceForm( + {"name": "LF", "service_type": "other", "sync_interval": "24", "setting_api_key": "k"}, + instance=service, + ) + self.assertTrue(form.fields["service_type"].disabled) + self.assertTrue(form.is_valid(), form.errors) + self.assertEqual(form.save().service_type, "lunchflow") + + def test_provider_settings_are_validated(self): + form = BankSyncServiceForm( + {"name": "LF", "service_type": "lunchflow", "sync_interval": "24"} + ) + self.assertFalse(form.is_valid()) + self.assertIn("setting_api_key", form.errors) + + def test_account_link_form(self): + eur = Currency.objects.create(code="€", name="Euro") + service = BankSyncService.objects.create(name="LF", service_type="lunchflow") + link = BankSyncAccount.objects.create( + service=service, external_id="7", available_currencies=["EUR", "USD"] + ) + account = Account.objects.create(name="EUR", currency=eur, owner=self.user) + + form = BankSyncAccountForm({"account": account.id}, instance=link) + self.assertFalse(form.is_valid()) + self.assertIn("start_date", form.errors) + + # Currency codes don't need to match: WYGIWYH's are free form + form = BankSyncAccountForm( + {"account": account.id, "start_date": "2026-10-01", "currencies": ["USD"]}, + instance=link, + ) + self.assertTrue(form.is_valid(), form.errors) + self.assertEqual(form.save().currencies, ["USD"]) + + form = BankSyncAccountForm({"currencies": ["GBP"]}, instance=link) + self.assertIn("currencies", form.errors) + + self.assertTrue(BankSyncAccountForm({}, instance=link).is_valid()) + + def test_linked_archived_account_stays_valid(self): + eur = Currency.objects.create(code="EUR", name="Euro") + service = BankSyncService.objects.create(name="LF", service_type="lunchflow") + archived = Account.objects.create(name="Old", currency=eur, owner=self.user, is_archived=True) + other_archived = Account.objects.create(name="Older", currency=eur, owner=self.user, is_archived=True) + link = BankSyncAccount.objects.create( + service=service, external_id="7", account=archived, start_date=date(2026, 10, 1) + ) + form = BankSyncAccountForm( + {"account": archived.id, "start_date": "2026-10-01"}, instance=link + ) + self.assertTrue(form.is_valid(), form.errors) + self.assertNotIn(other_archived, form.fields["account"].queryset) + + +class ViewTests(TestCase): + def setUp(self): + self.user = get_user_model().objects.create_superuser( + email="admin@example.com", password="x" + ) + self.client.force_login(self.user) + self.htmx = {"HTTP_HX_REQUEST": "true"} + + def test_switching_provider_renders_its_fields(self): + response = self.client.get( + reverse("bank_sync_add"), {"service_type": "lunchflow", "name": "Typed"}, **self.htmx + ) + self.assertContains(response, 'name="setting_api_key"') + self.assertContains(response, 'value="Typed"') + + def test_add_and_map_accounts(self): + response = self.client.post( + reverse("bank_sync_add"), + {"name": "LF", "service_type": "lunchflow", "sync_interval": "24", "setting_api_key": "k"}, + **self.htmx, + ) + self.assertEqual(response.status_code, 204) + service = BankSyncService.objects.get() + self.assertEqual(service.owner, self.user) + + provider = FakeProvider( + accounts=[ + RemoteAccount("7", "Bank - Checking", ("EUR",)), + RemoteAccount("8", "Card", (), "Connection expired"), + ] + ) + with patch.object(BankSyncService, "get_provider", return_value=provider): + response = self.client.get(reverse("bank_sync_accounts", args=[service.id]), **self.htmx) + self.assertContains(response, "Bank - Checking") + self.assertContains(response, "Connection expired") + self.assertContains(response, 'value="EUR"') + + account = Account.objects.create( + name="Checking", currency=Currency.objects.create(code="EUR", name="Euro") + ) + link = service.accounts.get(external_id="7") + card = service.accounts.get(external_id="8") + response = self.client.post( + reverse("bank_sync_accounts", args=[service.id]), + { + "form-TOTAL_FORMS": "2", + "form-INITIAL_FORMS": "2", + "form-0-id": card.id, + "form-1-id": link.id, + "form-1-account": account.id, + "form-1-start_date": "2026-10-01", + "form-1-currencies": "EUR", + }, + **self.htmx, + ) + self.assertEqual(response.status_code, 204) + link.refresh_from_db() + self.assertEqual(link.account, account) + self.assertEqual(link.currencies, ["EUR"]) + + def test_accounts_view_shows_provider_errors(self): + service = BankSyncService.objects.create(name="LF", service_type="lunchflow") + with patch.object(BankSyncService, "get_provider", side_effect=Exception("401")): + response = self.client.get(reverse("bank_sync_accounts", args=[service.id]), **self.htmx) + self.assertContains(response, "401") + + def test_other_users_services_are_hidden(self): + other = get_user_model().objects.create_user(email="o@example.com", password="x") + service = BankSyncService.objects.create(name="LF", service_type="lunchflow", owner=other) + response = self.client.get(reverse("bank_sync_edit", args=[service.id]), **self.htmx) + self.assertEqual(response.status_code, 404) + + +class ExportRestoreTests(TestCase): + def setUp(self): + self.admin = get_user_model().objects.create_superuser( + email="admin@example.com", password="x" + ) + self.other = get_user_model().objects.create_user( + email="other@example.com", password="x" + ) + self.client.force_login(self.admin) + eur = Currency.objects.create(code="EUR", name="Euro") + # Same name for both users: links must not mix them up + self.account = Account.objects.create(name="Checking", currency=eur, owner=self.other) + Account.objects.create(name="Checking", currency=eur, owner=self.admin) + self.service = BankSyncService.objects.create( + name="LF", + service_type="lunchflow", + owner=self.other, + settings={"api_key": "secret"}, + sync_interval=12, + trigger_rules=False, + match_existing=True, + ) + BankSyncAccount.objects.create( + service=self.service, + external_id="7", + external_name="Bank - Checking", + available_currencies=["EUR", "USD"], + currencies=["EUR"], + account=self.account, + start_date=date(2026, 10, 1), + last_synced_date=date(2026, 11, 1), + ) + BankSyncAccount.objects.create(service=self.service, external_id="8") + + def test_round_trip(self): + response = self.client.post( + reverse("export_form"), + {"users": "on", "accounts": "on", "currencies": "on", "bank_sync": "on"}, + HTTP_HX_REQUEST="true", + ) + self.assertEqual(response["Content-Type"], "application/zip") + with zipfile.ZipFile(io.BytesIO(response.content)) as z: + self.assertIn("bank_sync_services.csv", z.namelist()) + self.assertIn("bank_sync_accounts.csv", z.namelist()) + + BankSyncService._base_manager.all().delete() + + upload = SimpleUploadedFile("export.zip", response.content) + response = self.client.post( + reverse("restore_form"), {"zip_file": upload}, HTTP_HX_REQUEST="true" + ) + self.assertEqual(response.status_code, 204) + + service = BankSyncService._base_manager.get() + self.assertEqual(service.owner, self.other) + self.assertEqual(service.settings, {"api_key": "secret"}) + self.assertEqual(service.sync_interval, 12) + self.assertFalse(service.trigger_rules) + self.assertTrue(service.match_existing) + + linked = service.accounts.get(external_id="7") + self.assertEqual(linked.account, self.account) + self.assertEqual(linked.start_date, date(2026, 10, 1)) + self.assertEqual(linked.last_synced_date, date(2026, 11, 1)) + self.assertEqual(linked.available_currencies, ["EUR", "USD"]) + self.assertEqual(linked.currencies, ["EUR"]) + self.assertIsNone(service.accounts.get(external_id="8").account) diff --git a/app/apps/bank_sync/urls.py b/app/apps/bank_sync/urls.py new file mode 100644 index 00000000..2b2c536d --- /dev/null +++ b/app/apps/bank_sync/urls.py @@ -0,0 +1,21 @@ +from django.urls import path + +from . import views + +urlpatterns = [ + path("bank-sync/", views.bank_sync_index, name="bank_sync_index"), + path("bank-sync/list/", views.bank_sync_list, name="bank_sync_list"), + path("bank-sync/add/", views.bank_sync_add, name="bank_sync_add"), + path("bank-sync//edit/", views.bank_sync_edit, name="bank_sync_edit"), + path( + "bank-sync//delete/", views.bank_sync_delete, name="bank_sync_delete" + ), + path( + "bank-sync//accounts/", + views.bank_sync_accounts, + name="bank_sync_accounts", + ), + path( + "bank-sync//sync/", views.bank_sync_sync_now, name="bank_sync_sync_now" + ), +] diff --git a/app/apps/bank_sync/views.py b/app/apps/bank_sync/views.py new file mode 100644 index 00000000..2d2b5c73 --- /dev/null +++ b/app/apps/bank_sync/views.py @@ -0,0 +1,135 @@ +import logging + +from django.contrib import messages +from django.contrib.auth.decorators import login_required +from django.http import HttpResponse +from django.shortcuts import get_object_or_404, render +from django.urls import reverse +from django.utils.translation import gettext_lazy as _ +from django.views.decorators.http import require_http_methods + +from apps.bank_sync.forms import BankSyncAccountFormSet, BankSyncServiceForm +from apps.bank_sync.models import BankSyncService +from apps.bank_sync.sync import refresh_remote_accounts +from apps.bank_sync.tasks import manual_bank_sync +from apps.common.decorators.demo import disabled_on_demo +from apps.common.decorators.htmx import only_htmx + +logger = logging.getLogger(__name__) + +_CLOSE = {"HX-Trigger": "updated, hide_offcanvas"} + + +@login_required +@disabled_on_demo +@require_http_methods(["GET"]) +def bank_sync_index(request): + return render(request, "bank_sync/pages/index.html") + + +@only_htmx +@login_required +@disabled_on_demo +@require_http_methods(["GET"]) +def bank_sync_list(request): + services = BankSyncService.objects.prefetch_related("accounts") + return render(request, "bank_sync/fragments/list.html", {"services": services}) + + +def _service_form(request, instance=None): + url = request.path + if request.method == "POST": + return BankSyncServiceForm(request.POST, instance=instance, refresh_url=url) + # A GET with params comes from switching provider: keep what was typed + return BankSyncServiceForm( + instance=instance, initial=request.GET.dict() or None, refresh_url=url + ) + + +@only_htmx +@login_required +@disabled_on_demo +@require_http_methods(["GET", "POST"]) +def bank_sync_add(request): + form = _service_form(request) + if request.method == "POST" and form.is_valid(): + form.save() + messages.success(request, _("Service added successfully")) + return HttpResponse(status=204, headers=_CLOSE) + + return render( + request, + "bank_sync/fragments/form.html", + {"form": form, "title": _("Add bank sync service")}, + ) + + +@only_htmx +@login_required +@disabled_on_demo +@require_http_methods(["GET", "POST"]) +def bank_sync_edit(request, pk): + service = get_object_or_404(BankSyncService, id=pk) + form = _service_form(request, instance=service) + if request.method == "POST" and form.is_valid(): + form.save() + messages.success(request, _("Service updated successfully")) + return HttpResponse(status=204, headers=_CLOSE) + + return render( + request, + "bank_sync/fragments/form.html", + {"form": form, "title": _("Edit bank sync service")}, + ) + + +@only_htmx +@login_required +@disabled_on_demo +@require_http_methods(["DELETE"]) +def bank_sync_delete(request, pk): + get_object_or_404(BankSyncService, id=pk).delete() + messages.success(request, _("Service deleted successfully")) + return HttpResponse(status=204, headers=_CLOSE) + + +@only_htmx +@login_required +@disabled_on_demo +@require_http_methods(["GET", "POST"]) +def bank_sync_accounts(request, pk): + service = get_object_or_404(BankSyncService, id=pk) + error = None + + if request.method == "POST": + formset = BankSyncAccountFormSet(request.POST, queryset=service.accounts.all()) + if formset.is_valid(): + formset.save() + messages.success(request, _("Accounts updated successfully")) + return HttpResponse(status=204, headers=_CLOSE) + else: + try: + refresh_remote_accounts(service) + except Exception as e: + logger.warning(f"Could not list accounts for {service.name}: {e}") + error = _("Couldn't load the accounts from the provider: %(error)s") % { + "error": e + } + formset = BankSyncAccountFormSet(queryset=service.accounts.all()) + + return render( + request, + "bank_sync/fragments/accounts.html", + {"service": service, "formset": formset, "error": error}, + ) + + +@only_htmx +@login_required +@disabled_on_demo +@require_http_methods(["GET"]) +def bank_sync_sync_now(request, pk): + service = get_object_or_404(BankSyncService, id=pk) + manual_bank_sync.defer(service_id=service.id) + messages.success(request, _("Sync queued successfully")) + return HttpResponse(status=204, headers={"HX-Trigger": "toasts"}) diff --git a/app/templates/bank_sync/fragments/accounts.html b/app/templates/bank_sync/fragments/accounts.html new file mode 100644 index 00000000..7e098228 --- /dev/null +++ b/app/templates/bank_sync/fragments/accounts.html @@ -0,0 +1,37 @@ +{% extends 'extends/offcanvas.html' %} +{% load i18n %} +{% load crispy_forms_tags %} + +{% block title %}{% translate 'Accounts' %} - {{ service.name }}{% endblock %} + +{% block body %} +{% if error %} + +{% endif %} +
+ {% csrf_token %} + {{ formset.management_form }} + {% for form in formset %} +
+ + {{ form.instance.external_name|default:form.instance.external_id }} + #{{ form.instance.external_id }} + + {% if form.instance.remote_error %} + + {% elif form.instance.last_error %} + + {% endif %} + {{ form.id }} + {{ form.account|as_crispy_field }} + {{ form.start_date|as_crispy_field }} + {% if form.currencies.field.choices %}{{ form.currencies|as_crispy_field }}{% endif %} +
+ {% empty %} + + {% endfor %} + {% if formset.forms %} + + {% endif %} +
+{% endblock %} diff --git a/app/templates/bank_sync/fragments/form.html b/app/templates/bank_sync/fragments/form.html new file mode 100644 index 00000000..9e82b50d --- /dev/null +++ b/app/templates/bank_sync/fragments/form.html @@ -0,0 +1,10 @@ +{% extends 'extends/offcanvas.html' %} +{% load crispy_forms_tags %} + +{% block title %}{{ title }}{% endblock %} + +{% block body %} +
+ {% crispy form %} +
+{% endblock %} diff --git a/app/templates/bank_sync/fragments/list.html b/app/templates/bank_sync/fragments/list.html new file mode 100644 index 00000000..066ba7f1 --- /dev/null +++ b/app/templates/bank_sync/fragments/list.html @@ -0,0 +1,84 @@ +{% load i18n %} + + +
+
+
{% translate 'Bank Sync' %}
+
+ +
+
+ {% if services %} +
+ + + + + + + + + + + + + {% for service in services %} + + + + + + + + + {% endfor %} + +
{% translate 'Name' %}{% translate 'Service' %}{% translate 'Accounts' %}{% translate 'Last sync' %}
+
+ + + + + + + +
+
{% if service.is_active %}{% else %} + {% endif %} + {{ service.name }} + {% if service.failure_count > 0 %} + + + {{ service.failure_count }} + + {% endif %} + {{ service.get_service_type_display }}{% with linked=service.accounts.all %}{% for link in linked %}{% if link.account %}{% with error=link.remote_error|default:link.last_error %}{% if error %}{% endif %}{{ link.account.name }}{% endwith %}{% endif %}{% endfor %}{% endwith %}{{ service.last_sync|date:"SHORT_DATETIME_FORMAT" }}
+
+ {% else %} + + {% endif %} +
+
+
diff --git a/app/templates/bank_sync/pages/index.html b/app/templates/bank_sync/pages/index.html new file mode 100644 index 00000000..8c37ca62 --- /dev/null +++ b/app/templates/bank_sync/pages/index.html @@ -0,0 +1,8 @@ +{% extends "layouts/base.html" %} +{% load i18n %} + +{% block title %}{% translate 'Bank Sync' %}{% endblock %} + +{% block content %} +
+{% endblock %} From 77ee56f7a31110640dc9eb08d89de396fc3cca40 Mon Sep 17 00:00:00 2001 From: Herculino Trotta Date: Sun, 8 Nov 2026 12:39:17 -0300 Subject: [PATCH 2/6] feat(bank-sync): add sidebar entry --- app/templates/includes/sidebar.html | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/app/templates/includes/sidebar.html b/app/templates/includes/sidebar.html index 1edaf52b..2ed7d4a3 100644 --- a/app/templates/includes/sidebar.html +++ b/app/templates/includes/sidebar.html @@ -144,7 +144,7 @@ + active="tags_index||entities_index||categories_index||accounts_index||account_groups_index||currencies_index||exchange_rates_index||rules_index||import_profiles_index||automatic_exchange_rates_index||bank_sync_index||export_index||users_index"> + + {% if user.is_superuser %} From 25a2326cf66638aed36e9b7e82b6beeaf76fef8d Mon Sep 17 00:00:00 2001 From: Herculino Trotta Date: Sun, 8 Nov 2026 12:39:17 -0300 Subject: [PATCH 3/6] feat(export): include bank sync services and account links in export and restore --- app/apps/export_app/forms.py | 16 +++++++++ app/apps/export_app/resources/bank_sync.py | 39 ++++++++++++++++++++++ app/apps/export_app/views.py | 9 +++++ 3 files changed, 64 insertions(+) create mode 100644 app/apps/export_app/resources/bank_sync.py diff --git a/app/apps/export_app/forms.py b/app/apps/export_app/forms.py index 1581791d..f9b2d229 100644 --- a/app/apps/export_app/forms.py +++ b/app/apps/export_app/forms.py @@ -73,6 +73,13 @@ class ExportForm(forms.Form): label=_("Automatic Exchange Rates"), initial=False, ) + bank_sync = forms.BooleanField( + required=False, + widget=forms.CheckboxInput(), + label=_("Bank Sync"), + help_text=_("Includes the providers' API keys"), + initial=False, + ) rules = forms.BooleanField( required=False, widget=forms.CheckboxInput(), @@ -110,6 +117,7 @@ class ExportForm(forms.Form): "recurring_transactions", "exchange_rates_services", "exchange_rates", + "bank_sync", "rules", "dca", "import_profiles", @@ -140,6 +148,12 @@ class RestoreForm(forms.Form): required=False, label=_("Automatic Exchange Rates") ) exchange_rates = forms.FileField(required=False, label=_("Exchange Rates")) + bank_sync_services = forms.FileField( + required=False, label=_("Bank Sync Services") + ) + bank_sync_accounts = forms.FileField( + required=False, label=_("Bank Sync Accounts") + ) transaction_rules = forms.FileField(required=False, label=_("Transaction rules")) transaction_rules_actions = forms.FileField( required=False, label=_("Edit transaction action") @@ -171,6 +185,8 @@ class RestoreForm(forms.Form): "recurring_transactions", "automatic_exchange_rates", "exchange_rates", + "bank_sync_services", + "bank_sync_accounts", "transaction_rules", "transaction_rules_actions", "transaction_rules_update_or_create", diff --git a/app/apps/export_app/resources/bank_sync.py b/app/apps/export_app/resources/bank_sync.py new file mode 100644 index 00000000..b3a2d4db --- /dev/null +++ b/app/apps/export_app/resources/bank_sync.py @@ -0,0 +1,39 @@ +from import_export import fields, resources, widgets + +from apps.accounts.models import Account +from apps.bank_sync.models import BankSyncAccount, BankSyncService + + +class _AnyOwnerWidget(widgets.ForeignKeyWidget): + """By id, across every user. + + The default managers only see the restoring user's rows, and names can clash + between users. Ids survive a restore. + """ + + def get_queryset(self, value, row, *args, **kwargs): + return self.model._base_manager.all() + + +class BankSyncServiceResource(resources.ModelResource): + class Meta: + model = BankSyncService + + def get_queryset(self): + return BankSyncService._base_manager.all() + + +class BankSyncAccountResource(resources.ModelResource): + service = fields.Field( + attribute="service", + column_name="service", + widget=_AnyOwnerWidget(BankSyncService), + ) + account = fields.Field( + attribute="account", + column_name="account", + widget=_AnyOwnerWidget(Account), + ) + + class Meta: + model = BankSyncAccount diff --git a/app/apps/export_app/views.py b/app/apps/export_app/views.py index b60f4a0c..a74e4ae0 100644 --- a/app/apps/export_app/views.py +++ b/app/apps/export_app/views.py @@ -15,6 +15,10 @@ from tablib import Dataset from apps.common.decorators.htmx import only_htmx from apps.export_app.forms import ExportForm, RestoreForm from apps.export_app.resources.accounts import AccountResource +from apps.export_app.resources.bank_sync import ( + BankSyncAccountResource, + BankSyncServiceResource, +) from apps.export_app.resources.currencies import ( CurrencyResource, ExchangeRateResource, @@ -119,6 +123,9 @@ def export_form(request): ) if export_exchange_rates: exports.append((ExchangeRateResource().export(), "exchange_rates")) + if form.cleaned_data.get("bank_sync"): + exports.append((BankSyncServiceResource().export(), "bank_sync_services")) + exports.append((BankSyncAccountResource().export(), "bank_sync_accounts")) if export_rules: exports.append( (TransactionRuleResource().export(), "transaction_rules") @@ -235,6 +242,8 @@ def process_imports(request, cleaned_data): ("installment_plans", InstallmentPlanResource), ("recurring_transactions", RecurringTransactionResource), ("transactions", TransactionResource), + ("bank_sync_services", BankSyncServiceResource), + ("bank_sync_accounts", BankSyncAccountResource), ("dca_strategies", DCAStrategyResource), ("dca_entries", DCAEntryResource), ("import_profiles", ImportProfileResource), From 9238c48bb507e104bdd40d8fabcd5efc16ace10a Mon Sep 17 00:00:00 2001 From: Herculino Trotta Date: Sun, 8 Nov 2026 16:08:37 -0300 Subject: [PATCH 4/6] feat(bank-sync): add SimpleFIN provider Users paste a one-time setup token. On save it's claimed and the access URL it returns is stored; credentials go in a header, never the URL, so errors don't leak them. - Providers can validate or transform their settings (clean_settings), run by the service form once every other field is valid, so a token isn't spent on a form that fails anyway. - Providers can declare a minimum sync interval. SimpleFIN Bridge disables tokens that go much past ~24 requests a day, so it's 6 hours. - Tokens and access URLs are user input pointing the server at a host: only SimpleFIN Bridge is allowed unless SIMPLEFIN_ALLOWED_HOSTS says otherwise. Checked again on every request, for restored settings. - Transactions are fetched per account in 90-day windows, Bridge's limit per request. Pending ones are skipped. --- app/apps/bank_sync/forms.py | 35 +++- app/apps/bank_sync/migrations/0001_initial.py | 2 +- app/apps/bank_sync/models.py | 1 + app/apps/bank_sync/providers/__init__.py | 2 + app/apps/bank_sync/providers/base.py | 10 + app/apps/bank_sync/providers/simplefin.py | 182 ++++++++++++++++++ app/apps/bank_sync/tests/test_bank_sync.py | 138 +++++++++++++ 7 files changed, 361 insertions(+), 9 deletions(-) create mode 100644 app/apps/bank_sync/providers/simplefin.py diff --git a/app/apps/bank_sync/forms.py b/app/apps/bank_sync/forms.py index 924f0234..52d0bc85 100644 --- a/app/apps/bank_sync/forms.py +++ b/app/apps/bank_sync/forms.py @@ -41,9 +41,11 @@ class BankSyncServiceForm(forms.ModelForm): service_type = self.data.get("service_type") or self.initial.get( "service_type" ) - provider = PROVIDER_MAPPING.get(service_type) + self.provider = PROVIDER_MAPPING.get(service_type) self.setting_names = [] - for key, field in (provider.settings_fields if provider else {}).items(): + for key, field in ( + self.provider.settings_fields if self.provider else {} + ).items(): name = SETTING_PREFIX + key self.fields[name] = copy.deepcopy(field) if key in self.instance.settings: @@ -79,12 +81,29 @@ class BankSyncServiceForm(forms.ModelForm): ), ) - def save(self, commit=True): - self.instance.settings = { - name.removeprefix(SETTING_PREFIX): self.cleaned_data[name] - for name in self.setting_names - } - return super().save(commit) + def clean_sync_interval(self): + interval = self.cleaned_data["sync_interval"] + if self.provider and interval < self.provider.min_sync_interval: + raise forms.ValidationError( + _("This service can't sync more often than every %(hours)s hours.") + % {"hours": self.provider.min_sync_interval} + ) + return interval + + def clean(self): + cleaned_data = super().clean() + # Only once the rest is valid: a provider may spend a one-time token here + if self.provider and not self.errors: + try: + self.instance.settings = self.provider.clean_settings( + { + name.removeprefix(SETTING_PREFIX): cleaned_data[name] + for name in self.setting_names + } + ) + except forms.ValidationError as e: + self.add_error(None, e) + return cleaned_data class BankSyncAccountForm(forms.ModelForm): diff --git a/app/apps/bank_sync/migrations/0001_initial.py b/app/apps/bank_sync/migrations/0001_initial.py index 5f49ee9e..4fa180fc 100644 --- a/app/apps/bank_sync/migrations/0001_initial.py +++ b/app/apps/bank_sync/migrations/0001_initial.py @@ -21,7 +21,7 @@ class Migration(migrations.Migration): fields=[ ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), ('name', models.CharField(max_length=255, verbose_name='Name')), - ('service_type', models.CharField(choices=[('lunchflow', 'Lunch Flow')], max_length=255, verbose_name='Service Type')), + ('service_type', models.CharField(choices=[('lunchflow', 'Lunch Flow'), ('simplefin', 'SimpleFIN')], max_length=255, verbose_name='Service Type')), ('is_active', models.BooleanField(default=True, verbose_name='Active')), ('settings', models.JSONField(blank=True, default=dict, verbose_name='Settings')), ('sync_interval', models.PositiveIntegerField(default=24, help_text='Most banks only update once a day.', validators=[django.core.validators.MinValueValidator(1), django.core.validators.MaxValueValidator(168)], verbose_name='Sync every X hours')), diff --git a/app/apps/bank_sync/models.py b/app/apps/bank_sync/models.py index 079e4832..765dc118 100644 --- a/app/apps/bank_sync/models.py +++ b/app/apps/bank_sync/models.py @@ -8,6 +8,7 @@ from apps.common.models import OwnedObject, OwnedObjectManager class BankSyncService(OwnedObject): class ServiceType(models.TextChoices): LUNCHFLOW = "lunchflow", "Lunch Flow" + SIMPLEFIN = "simplefin", "SimpleFIN" name = models.CharField(max_length=255, verbose_name=_("Name")) service_type = models.CharField( diff --git a/app/apps/bank_sync/providers/__init__.py b/app/apps/bank_sync/providers/__init__.py index 1edb212d..a097ee9c 100644 --- a/app/apps/bank_sync/providers/__init__.py +++ b/app/apps/bank_sync/providers/__init__.py @@ -1,6 +1,8 @@ from apps.bank_sync.providers.lunchflow import LunchFlowProvider +from apps.bank_sync.providers.simplefin import SimpleFINProvider # Keys must match BankSyncService.ServiceType PROVIDER_MAPPING = { "lunchflow": LunchFlowProvider, + "simplefin": SimpleFINProvider, } diff --git a/app/apps/bank_sync/providers/base.py b/app/apps/bank_sync/providers/base.py index da1921c9..810a338e 100644 --- a/app/apps/bank_sync/providers/base.py +++ b/app/apps/bank_sync/providers/base.py @@ -34,10 +34,20 @@ class BankSyncProvider(ABC): """ settings_fields: dict = {} + # In hours, for services that limit how often they can be called + min_sync_interval: int = 1 def __init__(self, settings: dict): self.settings = settings or {} + @classmethod + def clean_settings(cls, settings: dict) -> dict: + """Validate or transform the settings before they're saved. + + Runs once every other form field is valid. Raise ValidationError to reject. + """ + return settings + @abstractmethod def list_accounts(self) -> list[RemoteAccount]: """Every account the credentials can see.""" diff --git a/app/apps/bank_sync/providers/simplefin.py b/app/apps/bank_sync/providers/simplefin.py new file mode 100644 index 00000000..4d0b5612 --- /dev/null +++ b/app/apps/bank_sync/providers/simplefin.py @@ -0,0 +1,182 @@ +import base64 +import binascii +import os +from datetime import date, datetime, time, timedelta +from decimal import Decimal +from urllib.parse import unquote, urlsplit, urlunsplit + +import requests +from django import forms +from django.utils import timezone +from django.utils.translation import gettext as _ +from django.utils.translation import gettext_lazy + +from apps.bank_sync.providers.base import ( + BankSyncProvider, + RemoteAccount, + RemoteTransaction, +) + +DEFAULT_ALLOWED_HOSTS = "bridge.simplefin.org,beta-bridge.simplefin.org" +# SimpleFIN Bridge refuses longer ranges in a single request +MAX_RANGE_DAYS = 90 + + +def _check_host(url: str) -> None: + # Tokens are user input: without this they could point the server at any host + allowed = { + host.strip().lower() + for host in os.getenv("SIMPLEFIN_ALLOWED_HOSTS", DEFAULT_ALLOWED_HOSTS).split(",") + if host.strip() + } + parts = urlsplit(url) + if parts.scheme not in ("http", "https") or (parts.hostname or "") not in allowed: + raise ValueError( + _("SimpleFIN server %(host)s isn't allowed on this instance.") + % {"host": parts.hostname or url} + ) + + +def _timestamp(day: date) -> int: + # Local midnight, so windows and transaction dates agree on what a day is + return int( + datetime.combine(day, time(), tzinfo=timezone.get_current_timezone()).timestamp() + ) + + +class SimpleFINProvider(BankSyncProvider): + """https://www.simplefin.org/protocol.html""" + + # SimpleFIN Bridge disables tokens that go much past ~24 requests a day, and a + # sync costs one plus one per account + min_sync_interval = 6 + + settings_fields = { + "access_url": forms.CharField( + label=gettext_lazy("Setup Token"), + widget=forms.TextInput(attrs={"autocomplete": "off"}), + help_text=gettext_lazy( + "Paste a setup token from your SimpleFIN server. It can only be " + "used once: on save it's exchanged for an access URL, which is " + "what you'll see here afterwards." + ), + ), + } + + @classmethod + def clean_settings(cls, settings: dict) -> dict: + try: + return cls._claim(settings) + except ValueError as e: + raise forms.ValidationError(str(e)) + + @staticmethod + def _claim(settings: dict) -> dict: + value = settings["access_url"].strip() + # Already claimed: the form is being saved again + if not value.startswith(("http://", "https://")): + try: + claim_url = base64.b64decode( + value + "=" * (-len(value) % 4), validate=True + ).decode() + except (binascii.Error, UnicodeDecodeError): + raise ValueError(_("This isn't a valid SimpleFIN setup token.")) + _check_host(claim_url) + try: + response = requests.post(claim_url, timeout=30) + response.raise_for_status() + except requests.RequestException as e: + raise ValueError( + _( + "Couldn't claim the setup token, it may have been used " + "already: %(error)s" + ) + % {"error": e} + ) + value = response.text.strip() + _check_host(value) + return {**settings, "access_url": value} + + def __init__(self, settings: dict): + super().__init__(settings) + parts = urlsplit(self.settings.get("access_url", "")) + self.session = requests.Session() + # Credentials go in a header, not the URL, so errors never show them + self.session.auth = (unquote(parts.username or ""), unquote(parts.password or "")) + netloc = (parts.hostname or "") + (f":{parts.port}" if parts.port else "") + self.base_url = urlunsplit((parts.scheme, netloc, parts.path.rstrip("/"), "", "")) + + def _get_accounts(self, **params) -> dict: + # Settings can also arrive through a backup restore, skipping the form + _check_host(self.base_url) + response = self.session.get( + f"{self.base_url}/accounts", + params={"version": "2", **params}, + # Slow: the server can fetch from every bank on each request + timeout=60, + ) + response.raise_for_status() + return response.json() + + def list_accounts(self) -> list[RemoteAccount]: + data = self._get_accounts(**{"balances-only": "1"}) + connections = {c["conn_id"]: c.get("name", "") for c in data.get("connections", [])} + account_errors, connection_errors = {}, {} + for error in data.get("errlist", []): + if error.get("account_id"): + account_errors.setdefault(error["account_id"], error.get("msg", "")) + elif error.get("conn_id"): + connection_errors.setdefault(error["conn_id"], error.get("msg", "")) + + accounts = [] + for account in data["accounts"]: + institution = connections.get(account.get("conn_id")) or ( + account.get("org") or {} + ).get("name") + currency = account.get("currency") or "" + accounts.append( + RemoteAccount( + id=str(account["id"]), + name=" - ".join(filter(None, [institution, account["name"]])), + # Custom currencies are URLs, nothing to match them against + currencies=(currency.upper(),) if currency and "/" not in currency else (), + error=account_errors.get(account["id"]) + or connection_errors.get(account.get("conn_id"), ""), + ) + ) + return accounts + + def get_transactions( + self, account_id: str, start_date: date + ) -> list[RemoteTransaction]: + transactions = [] + start, stop = start_date, timezone.localdate() + timedelta(days=1) + while start < stop: + end = min(start + timedelta(days=MAX_RANGE_DAYS), stop) + data = self._get_accounts( + account=account_id, + **{"start-date": _timestamp(start), "end-date": _timestamp(end)}, + ) + for account in data["accounts"]: + if str(account["id"]) != account_id: + continue + currency = account.get("currency") or "" + for tx in account.get("transactions", []): + if tx.get("pending"): + continue + payee = (tx.get("payee") or "").strip() + description = (tx.get("description") or "").strip() + transactions.append( + RemoteTransaction( + id=str(tx["id"]), + date=datetime.fromtimestamp( + tx["posted"], tz=timezone.get_current_timezone() + ).date(), + amount=Decimal(str(tx["amount"])), + description=payee or description, + notes=description if payee and description != payee else "", + currency=currency.upper() if "/" not in currency else "", + ) + ) + start = end + return transactions diff --git a/app/apps/bank_sync/tests/test_bank_sync.py b/app/apps/bank_sync/tests/test_bank_sync.py index 346bef10..104f01da 100644 --- a/app/apps/bank_sync/tests/test_bank_sync.py +++ b/app/apps/bank_sync/tests/test_bank_sync.py @@ -1,3 +1,4 @@ +import base64 import io import os import zipfile @@ -6,6 +7,7 @@ from decimal import Decimal from unittest.mock import MagicMock, patch from django.contrib.auth import get_user_model +from django.core.exceptions import ValidationError from django.core.files.uploadedfile import SimpleUploadedFile from django.test import TestCase from django.urls import reverse @@ -16,6 +18,8 @@ from apps.bank_sync.forms import BankSyncAccountForm, BankSyncServiceForm from apps.bank_sync.models import BankSyncAccount, BankSyncService from apps.bank_sync.providers.base import RemoteAccount, RemoteTransaction from apps.bank_sync.providers.lunchflow import LunchFlowProvider +from apps.bank_sync.providers.simplefin import SimpleFINProvider +from apps.bank_sync.providers.simplefin import _timestamp as _ts from apps.bank_sync.sync import is_due, refresh_remote_accounts, sync_service from apps.common.middleware.thread_local import delete_current_user, write_current_user from apps.currencies.models import Currency @@ -96,6 +100,115 @@ class LunchFlowProviderTests(TestCase): ) +class SimpleFINProviderTests(TestCase): + def setUp(self): + self.enterContext(patch.dict("os.environ")) + os.environ.pop("SIMPLEFIN_ALLOWED_HOSTS", None) + self.provider = SimpleFINProvider( + {"access_url": "https://us%40r:p%3Ass@bridge.simplefin.org/simplefin/"} + ) + + def test_credentials_are_kept_out_of_the_url(self): + self.assertEqual(self.provider.session.auth, ("us@r", "p:ss")) + self.assertEqual(self.provider.base_url, "https://bridge.simplefin.org/simplefin") + + def test_claims_setup_token(self): + token = base64.b64encode(b"https://bridge.simplefin.org/simplefin/claim/abc").decode() + response = MagicMock(text="https://u:p@bridge.simplefin.org/simplefin\n") + with patch("requests.post", return_value=response) as post: + settings = SimpleFINProvider.clean_settings({"access_url": token}) + post.assert_called_once_with( + "https://bridge.simplefin.org/simplefin/claim/abc", timeout=30 + ) + self.assertEqual(settings, {"access_url": "https://u:p@bridge.simplefin.org/simplefin"}) + + def test_claimed_url_is_not_claimed_again(self): + with patch("requests.post") as post: + SimpleFINProvider.clean_settings({"access_url": "https://u:p@bridge.simplefin.org/x"}) + post.assert_not_called() + + def test_rejects_hosts_not_allowed(self): + token = base64.b64encode(b"http://db:5432/claim").decode() + with patch("requests.post") as post: + for value in (token, "https://u:p@evil.test/x", "not base64!"): + with self.assertRaises(ValidationError): + SimpleFINProvider.clean_settings({"access_url": value}) + post.assert_not_called() + with patch.dict("os.environ", {"SIMPLEFIN_ALLOWED_HOSTS": "sfin.local"}): + SimpleFINProvider.clean_settings({"access_url": "http://u:p@sfin.local/x"}) + # Restored settings skip the form, the request itself is checked too + provider = SimpleFINProvider({"access_url": "https://u:p@evil.test/x"}) + with self.assertRaises(ValueError): + provider.list_accounts() + + def test_list_accounts(self): + payload = { + "errlist": [ + {"code": "con.auth", "msg": "Reconnect", "conn_id": "c2"}, + {"code": "gen.api", "msg": "Slow down"}, + ], + "connections": [{"conn_id": "c1", "name": "Bank"}, {"conn_id": "c2", "name": "Other"}], + "accounts": [ + {"id": "a1", "name": "Checking", "conn_id": "c1", "currency": "usd"}, + {"id": "a2", "name": "Card", "conn_id": "c2", "currency": "https://x.test/pts"}, + ], + } + with patch.object(self.provider.session, "get", return_value=_response(payload)) as get: + accounts = self.provider.list_accounts() + get.assert_called_once_with( + "https://bridge.simplefin.org/simplefin/accounts", + params={"version": "2", "balances-only": "1"}, + timeout=60, + ) + self.assertEqual( + accounts, + [ + RemoteAccount("a1", "Bank - Checking", ("USD",), ""), + RemoteAccount("a2", "Other - Card", (), "Reconnect"), + ], + ) + + def test_get_transactions_in_90_day_windows(self): + today = timezone.localdate() + start = today - timedelta(days=100) + + def get(url, params, timeout): + if params["start-date"] != _ts(start): + return _response({"accounts": []}) + return _response( + { + "accounts": [ + { + "id": "a1", + "currency": "USD", + "transactions": [ + {"id": "t1", "posted": _ts(start) + 3600, "amount": "-5.10", "description": "CARD COFFEE", "payee": "Coffee"}, + {"id": "t2", "posted": 0, "amount": "-1", "description": "Pending", "pending": True}, + ], + }, + {"id": "other", "transactions": [{"id": "x", "posted": 0, "amount": "1"}]}, + ] + } + ) + + with patch.object(self.provider.session, "get", side_effect=get) as mock_get: + transactions = self.provider.get_transactions("a1", start) + + windows = [(c.kwargs["params"]["start-date"], c.kwargs["params"]["end-date"]) for c in mock_get.call_args_list] + self.assertEqual( + windows, + [ + (_ts(start), _ts(start + timedelta(days=90))), + (_ts(start + timedelta(days=90)), _ts(today + timedelta(days=1))), + ], + ) + self.assertEqual(mock_get.call_args.kwargs["params"]["account"], "a1") + self.assertEqual( + transactions, + [RemoteTransaction("t1", start, Decimal("-5.10"), "Coffee", "CARD COFFEE", "USD")], + ) + + class FakeProvider: def __init__(self, transactions=(), accounts=()): self.transactions = list(transactions) @@ -401,6 +514,31 @@ class FormTests(TestCase): self.assertFalse(form.is_valid()) self.assertIn("setting_api_key", form.errors) + def test_provider_minimum_sync_interval(self): + data = {"name": "SF", "service_type": "simplefin", "setting_access_url": "x"} + with patch.object(SimpleFINProvider, "clean_settings", side_effect=lambda s: s): + form = BankSyncServiceForm({**data, "sync_interval": "5"}) + self.assertIn("sync_interval", form.errors) + self.assertTrue(BankSyncServiceForm({**data, "sync_interval": "6"}).is_valid()) + + @patch.object(SimpleFINProvider, "clean_settings") + def test_clean_settings_runs_last_and_its_errors_show(self, clean_settings): + data = {"service_type": "simplefin", "sync_interval": "24", "setting_access_url": "tok"} + # Another field is invalid: a one-time token must not be spent + self.assertFalse(BankSyncServiceForm(data).is_valid()) + clean_settings.assert_not_called() + + clean_settings.side_effect = ValidationError("Token used") + form = BankSyncServiceForm({**data, "name": "SF"}) + self.assertFalse(form.is_valid()) + self.assertEqual(form.non_field_errors(), ["Token used"]) + + clean_settings.side_effect = None + clean_settings.return_value = {"access_url": "https://u:p@bridge.simplefin.org/x"} + form = BankSyncServiceForm({**data, "name": "SF"}) + self.assertTrue(form.is_valid()) + self.assertEqual(form.save().settings, clean_settings.return_value) + def test_account_link_form(self): eur = Currency.objects.create(code="€", name="Euro") service = BankSyncService.objects.create(name="LF", service_type="lunchflow") From 2a4202118bdc9025313f03211aaaa288aff0026e Mon Sep 17 00:00:00 2001 From: Herculino Trotta Date: Sun, 8 Nov 2026 16:09:23 -0300 Subject: [PATCH 5/6] feat(bank-sync): import pending transactions and settle them when they post Providers can now return pending transactions, flagged as such, when their ids survive posting. They're imported unpaid right away and, once the bank posts them, marked paid with their final amount and date. - Settling happens once, so later edits stick, and one marked paid by hand is left alone. It goes through save() so a transfer's other side follows; no rules run again. reference_date moves only if it still followed the date. - Each sync looks back to the oldest unpaid synced transaction, since a card purchase can post weeks later when the bill closes. Pending ones older than 60 days are taken as cancelled and no longer followed. --- app/apps/bank_sync/providers/base.py | 3 + app/apps/bank_sync/sync.py | 53 +++++++++++++--- app/apps/bank_sync/tests/test_bank_sync.py | 70 ++++++++++++++++++++++ 3 files changed, 119 insertions(+), 7 deletions(-) diff --git a/app/apps/bank_sync/providers/base.py b/app/apps/bank_sync/providers/base.py index 810a338e..fc1b9168 100644 --- a/app/apps/bank_sync/providers/base.py +++ b/app/apps/bank_sync/providers/base.py @@ -23,6 +23,9 @@ class RemoteTransaction: description: str = "" notes: str = "" currency: str = "" + # Not settled yet. Imported unpaid and updated when it posts, so only return + # these when the provider keeps the id once they post + pending: bool = False class BankSyncProvider(ABC): diff --git a/app/apps/bank_sync/sync.py b/app/apps/bank_sync/sync.py index f3e95ed3..3961c7b1 100644 --- a/app/apps/bank_sync/sync.py +++ b/app/apps/bank_sync/sync.py @@ -1,6 +1,7 @@ import logging from datetime import timedelta +from django.db.models import Min from django.utils import timezone from apps.bank_sync.models import BankSyncAccount, BankSyncService @@ -15,6 +16,9 @@ logger = logging.getLogger(__name__) OVERLAP_DAYS = 7 # How far apart a synced transaction and the one it matches can be dated MATCH_DAYS = 3 +# Pending ones older than this are taken as cancelled and no longer followed. A +# card's bill cycle is at most ~40 days +PENDING_MAX_DAYS = 60 def refresh_remote_accounts(service: BankSyncService, provider=None) -> None: @@ -58,10 +62,39 @@ def _match_existing(link: BankSyncAccount, internal_id, type_, amount, date) -> return True +def _settle(transaction: Transaction, amount, date) -> None: + """The bank posted a transaction that was still unpaid: take its final values. + + Happens once, so later edits stick. One marked paid by hand is left alone. + """ + if transaction.is_paid or transaction.deleted: + return + # Only when it still follows the date, so one picked by the user is kept + if transaction.reference_date == transaction.date.replace(day=1): + transaction.reference_date = date.replace(day=1) + transaction.is_paid = True + transaction.amount = amount + transaction.date = date + # save() keeps a transfer's other side in step. It sends no signal, and as a + # bank update rather than a user edit, rules don't run again + transaction.save() + + def _sync_account(service: BankSyncService, link: BankSyncAccount, provider) -> int: + prefix = f"{service.service_type}:{link.external_id}:" start = link.start_date if link.last_synced_date: start = max(start, link.last_synced_date - timedelta(days=OVERLAP_DAYS)) + # Pending ones can post weeks later (a card's bill closing): look back far + # enough to see it happen + oldest_unpaid = Transaction.userless_all_objects.filter( + internal_id__startswith=prefix, + is_paid=False, + deleted=False, + date__gte=timezone.localdate() - timedelta(days=PENDING_MAX_DAYS), + ).aggregate(Min("date"))["date__min"] + if oldest_unpaid: + start = min(start, oldest_unpaid) created = 0 seen_currencies = set() @@ -73,16 +106,22 @@ def _sync_account(service: BankSyncService, link: BankSyncAccount, provider) -> if link.currencies and remote.currency not in link.currencies: continue - internal_id = f"{service.service_type}:{link.external_id}:{remote.id}" - # userless_all_objects: also sees soft deleted ones, so deleting a synced - # transaction doesn't bring it back on the next run - if Transaction.userless_all_objects.filter(internal_id=internal_id).exists(): - continue - + internal_id = prefix + remote.id type_ = ( Transaction.Type.INCOME if remote.amount > 0 else Transaction.Type.EXPENSE ) amount = abs(remote.amount) + + # userless_all_objects: also sees soft deleted ones, so deleting a synced + # transaction doesn't bring it back on the next run + existing = Transaction.userless_all_objects.filter( + internal_id=internal_id + ).first() + if existing: + if not remote.pending: + _settle(existing, amount, remote.date) + continue + if service.match_existing and _match_existing( link, internal_id, type_, amount, remote.date ): @@ -96,7 +135,7 @@ def _sync_account(service: BankSyncService, link: BankSyncAccount, provider) -> date=remote.date, description=remote.description[:500], notes=remote.notes, - is_paid=True, + is_paid=not remote.pending, internal_id=internal_id, ) created += 1 diff --git a/app/apps/bank_sync/tests/test_bank_sync.py b/app/apps/bank_sync/tests/test_bank_sync.py index 104f01da..a74a7f25 100644 --- a/app/apps/bank_sync/tests/test_bank_sync.py +++ b/app/apps/bank_sync/tests/test_bank_sync.py @@ -295,6 +295,76 @@ class SyncTests(TestCase): self._sync() self.assertEqual(self.provider.calls, [("7", date(2026, 10, 25))]) + def _pending(self, days_ago, amount="-20", pending=True): + day = timezone.localdate() - timedelta(days=days_ago) + return RemoteTransaction("p", day, Decimal(amount), "Hotel", pending=pending) + + def _hotel(self): + return Transaction.userless_all_objects.get(internal_id="lunchflow:7:p") + + def test_pending_is_imported_unpaid_and_settled_once_when_it_posts(self): + self.link.start_date = timezone.localdate() - timedelta(days=90) + self.link.save() + self.provider.transactions = [self._pending(30)] + self.assertEqual(self._sync(), 1) + self.assertFalse(self._hotel().is_paid) + + # Still pending with another amount: left alone, edits stick + self.provider.transactions = [self._pending(30, "-22")] + self._sync() + self.assertEqual(self._hotel().amount, Decimal("20")) + # And it's still looked for, way past the overlap + self.assertEqual(self.provider.calls[-1][1], timezone.localdate() - timedelta(days=30)) + + # Posts with its final amount and date + self.provider.transactions = [self._pending(29, "-25.509", pending=False)] + self.assertEqual(self._sync(), 0) + hotel = self._hotel() + self.assertTrue(hotel.is_paid) + self.assertEqual(hotel.amount, Decimal("25.50")) + self.assertEqual(hotel.date, timezone.localdate() - timedelta(days=29)) + self.assertEqual(hotel.reference_date, hotel.date.replace(day=1)) + + # Settled: later changes on the bank don't touch it, look back is normal + self.provider.transactions = [self._pending(29, "-99", pending=False)] + self._sync() + self.assertEqual(self._hotel().amount, Decimal("25.50")) + self.assertEqual(self.provider.calls[-1][1], timezone.localdate() - timedelta(days=7)) + + def test_settling_keeps_a_reference_date_picked_by_the_user(self): + self.link.start_date = timezone.localdate() - timedelta(days=90) + self.link.save() + self.provider.transactions = [self._pending(10)] + self._sync() + Transaction.userless_all_objects.filter(internal_id="lunchflow:7:p").update( + emit_signal=False, reference_date=date(2030, 1, 1) + ) + self.provider.transactions = [self._pending(10, pending=False)] + self._sync() + self.assertEqual(self._hotel().reference_date, date(2030, 1, 1)) + self.assertTrue(self._hotel().is_paid) + + def test_unpaid_marked_paid_by_hand_is_left_alone(self): + self.link.start_date = timezone.localdate() - timedelta(days=90) + self.link.save() + self.provider.transactions = [self._pending(10)] + self._sync() + Transaction.userless_all_objects.filter(internal_id="lunchflow:7:p").update( + emit_signal=False, is_paid=True + ) + self.provider.transactions = [self._pending(10, "-99", pending=False)] + self._sync() + self.assertEqual(self._hotel().amount, Decimal("20")) + + def test_old_pending_is_no_longer_followed(self): + self.link.start_date = timezone.localdate() - timedelta(days=200) + self.link.save() + self.provider.transactions = [self._pending(61)] + self._sync() + self._sync() + # Taken as cancelled: back to the normal overlap + self.assertEqual(self.provider.calls[-1][1], timezone.localdate() - timedelta(days=7)) + def test_unlinked_accounts_are_skipped(self): self.link.account = None self.link.save() From cc4c58b676c8ce2419b59650ac72336b56940081 Mon Sep 17 00:00:00 2001 From: Herculino Trotta Date: Sun, 8 Nov 2026 16:10:38 -0300 Subject: [PATCH 6/6] feat(bank-sync): add Pluggy provider Brazilian banks through Open Finance. Users give their Pluggy client credentials and the ids of their connections (items), from Meu Pluggy. - Uses the cursor-paginated /v2/transactions; /transactions is removed after 2026-12-31. Our own filters are sent again with each cursor. - The direction comes from `type`, not the amount's sign: credit cards keep the card's sign, purchases are positive. Checked against a real Meu Pluggy card and checking account; the sandbox data doesn't match production here. - Pending transactions keep their id once they post (a card's open bill stays pending until it closes), so they're imported unpaid and settled later. Future-dated ones (installments, scheduled payments) wait for their date. Installments already come dated in their own month. - Foreign purchases use the account's amount, with the original amount in the notes. - Connections that need the user to log in again show as an error. --- app/apps/bank_sync/migrations/0001_initial.py | 2 +- app/apps/bank_sync/models.py | 1 + app/apps/bank_sync/providers/__init__.py | 2 + app/apps/bank_sync/providers/pluggy.py | 147 ++++++++++++++++++ app/apps/bank_sync/tests/test_bank_sync.py | 106 +++++++++++++ 5 files changed, 257 insertions(+), 1 deletion(-) create mode 100644 app/apps/bank_sync/providers/pluggy.py diff --git a/app/apps/bank_sync/migrations/0001_initial.py b/app/apps/bank_sync/migrations/0001_initial.py index 4fa180fc..e8b2a3a3 100644 --- a/app/apps/bank_sync/migrations/0001_initial.py +++ b/app/apps/bank_sync/migrations/0001_initial.py @@ -21,7 +21,7 @@ class Migration(migrations.Migration): fields=[ ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), ('name', models.CharField(max_length=255, verbose_name='Name')), - ('service_type', models.CharField(choices=[('lunchflow', 'Lunch Flow'), ('simplefin', 'SimpleFIN')], max_length=255, verbose_name='Service Type')), + ('service_type', models.CharField(choices=[('lunchflow', 'Lunch Flow'), ('simplefin', 'SimpleFIN'), ('pluggy', 'Pluggy')], max_length=255, verbose_name='Service Type')), ('is_active', models.BooleanField(default=True, verbose_name='Active')), ('settings', models.JSONField(blank=True, default=dict, verbose_name='Settings')), ('sync_interval', models.PositiveIntegerField(default=24, help_text='Most banks only update once a day.', validators=[django.core.validators.MinValueValidator(1), django.core.validators.MaxValueValidator(168)], verbose_name='Sync every X hours')), diff --git a/app/apps/bank_sync/models.py b/app/apps/bank_sync/models.py index 765dc118..7a66d14b 100644 --- a/app/apps/bank_sync/models.py +++ b/app/apps/bank_sync/models.py @@ -9,6 +9,7 @@ class BankSyncService(OwnedObject): class ServiceType(models.TextChoices): LUNCHFLOW = "lunchflow", "Lunch Flow" SIMPLEFIN = "simplefin", "SimpleFIN" + PLUGGY = "pluggy", "Pluggy" name = models.CharField(max_length=255, verbose_name=_("Name")) service_type = models.CharField( diff --git a/app/apps/bank_sync/providers/__init__.py b/app/apps/bank_sync/providers/__init__.py index a097ee9c..ba03c68f 100644 --- a/app/apps/bank_sync/providers/__init__.py +++ b/app/apps/bank_sync/providers/__init__.py @@ -1,8 +1,10 @@ from apps.bank_sync.providers.lunchflow import LunchFlowProvider +from apps.bank_sync.providers.pluggy import PluggyProvider from apps.bank_sync.providers.simplefin import SimpleFINProvider # Keys must match BankSyncService.ServiceType PROVIDER_MAPPING = { "lunchflow": LunchFlowProvider, "simplefin": SimpleFINProvider, + "pluggy": PluggyProvider, } diff --git a/app/apps/bank_sync/providers/pluggy.py b/app/apps/bank_sync/providers/pluggy.py new file mode 100644 index 00000000..89320f96 --- /dev/null +++ b/app/apps/bank_sync/providers/pluggy.py @@ -0,0 +1,147 @@ +import re +from datetime import date +from decimal import Decimal +from urllib.parse import parse_qs, urlsplit + +import requests +from django import forms +from django.utils import timezone +from django.utils.translation import gettext as _ +from django.utils.translation import gettext_lazy + +from apps.bank_sync.providers.base import ( + BankSyncProvider, + RemoteAccount, + RemoteTransaction, +) + +BASE_URL = "https://api.pluggy.ai" +# Item statuses that need the user to reconnect on Pluggy +NEEDS_ACTION = {"LOGIN_ERROR", "WAITING_USER_INPUT"} + + +class PluggyProvider(BankSyncProvider): + """https://docs.pluggy.ai""" + + settings_fields = { + "client_id": forms.CharField( + label=gettext_lazy("Client ID"), + widget=forms.TextInput(attrs={"autocomplete": "off"}), + ), + "client_secret": forms.CharField( + label=gettext_lazy("Client Secret"), + # Not a PasswordInput: browsers autofill those with saved logins + widget=forms.TextInput(attrs={"autocomplete": "off"}), + ), + "item_ids": forms.CharField( + label=gettext_lazy("Item IDs"), + widget=forms.TextInput(attrs={"autocomplete": "off"}), + help_text=gettext_lazy( + "The IDs of your Pluggy connections (items), separated by commas. " + "Meu Pluggy shows them for each connected institution." + ), + ), + } + + def __init__(self, settings: dict): + super().__init__(settings) + self.session = requests.Session() + self.item_ids = [ + item_id + for item_id in re.split(r"[\s,]+", self.settings.get("item_ids", "")) + if item_id + ] + + def _get(self, path: str, **params) -> dict: + if "X-API-KEY" not in self.session.headers: + # Lasts 2 hours, longer than any sync + response = self.session.post( + f"{BASE_URL}/auth", + json={ + "clientId": self.settings.get("client_id", ""), + "clientSecret": self.settings.get("client_secret", ""), + }, + timeout=30, + ) + response.raise_for_status() + self.session.headers["X-API-KEY"] = response.json()["apiKey"] + response = self.session.get(f"{BASE_URL}{path}", params=params, timeout=30) + response.raise_for_status() + return response.json() + + def list_accounts(self) -> list[RemoteAccount]: + accounts = [] + for item_id in self.item_ids: + item = self._get(f"/items/{item_id}") + institution = (item.get("connector") or {}).get("name") + status = item.get("status") or "" + error = ( + _( + "Pluggy reports this connection as %(status)s, reconnect it " + "on Pluggy." + ) + % {"status": status} + if status in NEEDS_ACTION + else "" + ) + for account in self._get("/accounts", itemId=item_id)["results"]: + currency = account.get("currencyCode") or "" + accounts.append( + RemoteAccount( + id=str(account["id"]), + name=" - ".join( + filter( + None, + [ + institution, + account.get("marketingName") or account["name"], + account.get("number"), + ], + ) + ), + currencies=(currency.upper(),) if currency else (), + error=error, + ) + ) + return accounts + + def get_transactions( + self, account_id: str, start_date: date + ) -> list[RemoteTransaction]: + params = {"accountId": account_id, "dateFrom": start_date.isoformat()} + transactions = [] + while True: + data = self._get("/v2/transactions", **params) + for tx in data["results"]: + day = date.fromisoformat(tx["date"][:10]) + # Pending keeps its id once it posts (a card's open bill). Future + # ones (installments, scheduled payments) wait for their date + pending = tx.get("status") == "PENDING" + if pending and day > timezone.localdate(): + continue + converted = tx.get("amountInAccountCurrency") + amount = abs(Decimal(str(tx["amount"] if converted is None else converted))) + notes = "" + if converted is not None: + # Foreign purchase: keep the account's amount, note the original + notes = f"{tx.get('currencyCode', '')} {abs(Decimal(str(tx['amount'])))}" + transactions.append( + RemoteTransaction( + id=str(tx["id"]), + # Installments already come dated in their own month + date=day, + # Not the amount's sign: cards keep theirs, purchases are + # positive. `type` is normalized for every account + amount=amount if tx.get("type") == "CREDIT" else -amount, + description=(tx.get("description") or "").strip(), + notes=notes, + currency="" + if converted is not None + else (tx.get("currencyCode") or "").upper(), + pending=pending, + ) + ) + if not data.get("next"): + return transactions + # Send our own filters again with the cursor, `next` may drop them + params["after"] = parse_qs(urlsplit(data["next"]).query)["after"][0] diff --git a/app/apps/bank_sync/tests/test_bank_sync.py b/app/apps/bank_sync/tests/test_bank_sync.py index a74a7f25..c9f28d9c 100644 --- a/app/apps/bank_sync/tests/test_bank_sync.py +++ b/app/apps/bank_sync/tests/test_bank_sync.py @@ -18,6 +18,7 @@ from apps.bank_sync.forms import BankSyncAccountForm, BankSyncServiceForm from apps.bank_sync.models import BankSyncAccount, BankSyncService from apps.bank_sync.providers.base import RemoteAccount, RemoteTransaction from apps.bank_sync.providers.lunchflow import LunchFlowProvider +from apps.bank_sync.providers.pluggy import PluggyProvider from apps.bank_sync.providers.simplefin import SimpleFINProvider from apps.bank_sync.providers.simplefin import _timestamp as _ts from apps.bank_sync.sync import is_due, refresh_remote_accounts, sync_service @@ -209,6 +210,111 @@ class SimpleFINProviderTests(TestCase): ) +class PluggyProviderTests(TestCase): + def setUp(self): + self.provider = PluggyProvider( + {"client_id": "id", "client_secret": "secret", "item_ids": "i1, i2\n"} + ) + auth = _response({"apiKey": "key"}) + self.post = self.enterContext( + patch.object(self.provider.session, "post", return_value=auth) + ) + + def test_list_accounts(self): + def get(url, params, timeout): + self.assertEqual(self.provider.session.headers["X-API-KEY"], "key") + if url.endswith("/items/i1"): + return _response({"status": "UPDATED", "connector": {"name": "Nubank"}}) + if url.endswith("/items/i2"): + return _response({"status": "LOGIN_ERROR", "connector": {"name": "Itaú"}}) + return _response( + { + "results": [ + {"id": f"{params['itemId']}-a", "name": "Conta", "marketingName": None, "number": "0001", "currencyCode": "BRL"} + ] + } + ) + + with patch.object(self.provider.session, "get", side_effect=get): + accounts = self.provider.list_accounts() + + self.post.assert_called_once_with( + "https://api.pluggy.ai/auth", + json={"clientId": "id", "clientSecret": "secret"}, + timeout=30, + ) + self.assertEqual(accounts[0], RemoteAccount("i1-a", "Nubank - Conta - 0001", ("BRL",), "")) + self.assertEqual(accounts[1].id, "i2-a") + self.assertIn("LOGIN_ERROR", accounts[1].error) + + def _get_transactions(self, pages): + responses = [_response(p) for p in pages] + with patch.object(self.provider.session, "get", side_effect=responses) as get: + transactions = self.provider.get_transactions("a", date(2026, 10, 1)) + return transactions, get.call_args_list + + def test_bank_account_follows_cursor_with_own_filters(self): + pages = [ + { + "results": [ + {"id": "t1", "date": "2026-10-01T03:00:00.000Z", "amount": -12.5, "type": "DEBIT", "status": "POSTED", "description": "Padaria", "currencyCode": "BRL"}, + {"id": "t2", "date": "2026-10-02T03:00:00.000Z", "amount": -1, "type": "DEBIT", "status": "PENDING", "description": "Pending"}, + ], + "next": "?accountId=a&after=MjAy%3D", + }, + { + "results": [ + {"id": "t3", "date": "2026-10-04T03:00:00.000Z", "amount": 100, "type": "CREDIT", "status": "POSTED", "description": "Pix", "currencyCode": "BRL"}, + ], + "next": None, + }, + ] + transactions, calls = self._get_transactions(pages) + + self.assertEqual( + [c.kwargs["params"] for c in calls], + [ + {"accountId": "a", "dateFrom": "2026-10-01"}, + {"accountId": "a", "dateFrom": "2026-10-01", "after": "MjAy="}, + ], + ) + self.assertEqual( + transactions, + [ + RemoteTransaction("t1", date(2026, 10, 1), Decimal("-12.5"), "Padaria", "", "BRL"), + RemoteTransaction("t2", date(2026, 10, 2), Decimal("-1"), "Pending", "", "", pending=True), + RemoteTransaction("t3", date(2026, 10, 4), Decimal("100"), "Pix", "", "BRL"), + ], + ) + + def test_credit_card_follows_type_not_sign(self): + # Shapes from a real Meu Pluggy card: purchases are positive DEBITs, + # the bill payment a negative CREDIT + pages = [ + { + "results": [ + {"id": "t1", "date": "2026-10-03T03:00:00.000Z", "amount": 10, "amountInAccountCurrency": 55.3, "type": "DEBIT", "status": "POSTED", "description": "Steam", "currencyCode": "USD"}, + {"id": "t2", "date": "2026-10-05T03:00:00.000Z", "amount": -500, "type": "CREDIT", "status": "POSTED", "description": "Pagamento recebido", "currencyCode": "BRL"}, + {"id": "t3", "date": "2026-10-13T03:00:00.000Z", "amount": 18.72, "type": "DEBIT", "status": "POSTED", "description": "Loja 2/10", "currencyCode": "BRL", + "creditCardMetadata": {"installmentNumber": 2, "totalInstallments": 10, "purchaseDate": "2026-09-12T03:00:00.000Z"}}, + # Future installment, waits for its date + {"id": "t4", "date": "2027-07-13T03:00:00.000Z", "amount": 18.72, "type": "DEBIT", "status": "PENDING", "description": "Loja 10/10", "currencyCode": "BRL"}, + ], + "next": None, + }, + ] + transactions, _ = self._get_transactions(pages) + + self.assertEqual( + transactions, + [ + RemoteTransaction("t1", date(2026, 10, 3), Decimal("-55.3"), "Steam", "USD 10", ""), + RemoteTransaction("t2", date(2026, 10, 5), Decimal("500"), "Pagamento recebido", "", "BRL"), + RemoteTransaction("t3", date(2026, 10, 13), Decimal("-18.72"), "Loja 2/10", "", "BRL"), + ], + ) + + class FakeProvider: def __init__(self, transactions=(), accounts=()): self.transactions = list(transactions)