Files
WYGIWYH/app/apps/bank_sync/sync.py
T
Herculino Trotta 620e7df015 feat(bank-sync): sync bank transactions from external providers
Provider registry modeled after automatic exchange rates: each provider
declares its own settings fields, rendered dynamically in the service form.
Lunch Flow is the first provider.

- Map provider accounts to WYGIWYH accounts with a start date and an
  optional currency filter
- Hourly task syncs due services; manual sync from the list
- Deduplicates through Transaction.internal_id, optionally matching
  transactions already entered by hand
- Per account errors and provider reported connection problems

Refs #498
2026-11-08 12:39:16 -03:00

173 lines
6.4 KiB
Python

import logging
from datetime import timedelta
from django.utils import timezone
from apps.bank_sync.models import BankSyncAccount, BankSyncService
from apps.common.functions.decimals import truncate_decimal
from apps.common.middleware.thread_local import delete_current_user, write_current_user
from apps.rules.signals import transaction_created
from apps.transactions.models import Transaction
logger = logging.getLogger(__name__)
# Banks post late and back-date transactions, so every sync looks this far back
OVERLAP_DAYS = 7
# How far apart a synced transaction and the one it matches can be dated
MATCH_DAYS = 3
def refresh_remote_accounts(service: BankSyncService, provider=None) -> None:
"""Mirror the provider's account list, keeping existing links.
Accounts the provider no longer returns are dropped: their ids are gone for good.
"""
provider = provider or service.get_provider()
remote_accounts = {remote.id: remote for remote in provider.list_accounts()}
service.accounts.exclude(external_id__in=remote_accounts).delete()
for remote in remote_accounts.values():
link, _ = BankSyncAccount.objects.get_or_create(
service=service, external_id=remote.id
)
link.external_name = remote.name
link.remote_error = remote.error
link.available_currencies = sorted(
set(link.available_currencies) | set(remote.currencies)
)
link.save()
def _match_existing(link: BankSyncAccount, internal_id, type_, amount, date) -> bool:
"""Tag a transaction the user already added instead of creating a new one."""
candidates = Transaction.userless_all_objects.filter(
account=link.account,
type=type_,
amount=truncate_decimal(amount, link.account.currency.decimal_places),
date__range=(date - timedelta(days=MATCH_DAYS), date + timedelta(days=MATCH_DAYS)),
internal_id__isnull=True,
deleted=False,
)
best = min(candidates, key=lambda t: abs((t.date - date).days), default=None)
if best is None:
return False
# Not a user edit, so no transaction_updated (it would run "on update" rules)
Transaction.userless_all_objects.filter(pk=best.pk).update(
emit_signal=False, internal_id=internal_id
)
return True
def _sync_account(service: BankSyncService, link: BankSyncAccount, provider) -> int:
start = link.start_date
if link.last_synced_date:
start = max(start, link.last_synced_date - timedelta(days=OVERLAP_DAYS))
created = 0
seen_currencies = set()
for remote in provider.get_transactions(link.external_id, start):
if remote.date < link.start_date:
continue
if remote.currency:
seen_currencies.add(remote.currency)
if link.currencies and remote.currency not in link.currencies:
continue
internal_id = f"{service.service_type}:{link.external_id}:{remote.id}"
# userless_all_objects: also sees soft deleted ones, so deleting a synced
# transaction doesn't bring it back on the next run
if Transaction.userless_all_objects.filter(internal_id=internal_id).exists():
continue
type_ = (
Transaction.Type.INCOME if remote.amount > 0 else Transaction.Type.EXPENSE
)
amount = abs(remote.amount)
if service.match_existing and _match_existing(
link, internal_id, type_, amount, remote.date
):
continue
transaction = Transaction.objects.create(
account=link.account,
owner=service.owner,
type=type_,
amount=amount,
date=remote.date,
description=remote.description[:500],
notes=remote.notes,
is_paid=True,
internal_id=internal_id,
)
created += 1
if service.trigger_rules:
transaction_created.send(sender=transaction)
link.last_synced_date = timezone.localdate()
link.available_currencies = sorted(set(link.available_currencies) | seen_currencies)
link.save(update_fields=["last_synced_date", "available_currencies"])
return created
def sync_service(service: BankSyncService) -> int:
"""Import new transactions for every linked account. Returns how many.
A failing account doesn't stop the others. Any failure keeps last_sync as is,
so the service is retried on the next run.
"""
created = 0
errors = []
# Rules and the owned-object managers read the user from the thread
write_current_user(service.owner)
try:
provider = service.get_provider()
refresh_remote_accounts(service, provider)
links = service.accounts.filter(
account__isnull=False, start_date__isnull=False
).select_related("account__currency")
for link in links:
error = link.remote_error
if not error:
try:
created += _sync_account(service, link, provider)
except Exception as e:
logger.exception(f"Bank sync failed for {service.name} / {link}")
error = str(e)
link.last_error = error[:1000]
link.save(update_fields=["last_error"])
if error:
errors.append(f"{link}: {error}")
except Exception as e:
logger.exception(f"Bank sync failed for {service.name}")
errors.append(str(e))
finally:
delete_current_user()
if errors:
service.failure_count += 1
service.last_error = "\n".join(errors)[:1000]
service.save(update_fields=["failure_count", "last_error"])
else:
service.last_sync = timezone.now()
service.failure_count = 0
service.last_error = ""
service.save(update_fields=["last_sync", "failure_count", "last_error"])
logger.info(f"Bank sync for {service.name} created {created} transactions")
return created
def is_due(service: BankSyncService) -> bool:
if service.last_sync is None:
return True
# Hour granularity, the scheduler runs on the hour
elapsed = timezone.now().replace(minute=0, second=0, microsecond=0) - (
service.last_sync.replace(minute=0, second=0, microsecond=0)
)
return elapsed >= timedelta(hours=service.sync_interval)
def sync_due_services(force: bool = False) -> None:
for service in BankSyncService.objects.filter(is_active=True):
if force or is_due(service):
sync_service(service)