Files
WYGIWYH/app/apps/bank_sync/sync.py
T
Herculino Trotta 2a4202118b feat(bank-sync): import pending transactions and settle them when they post
Providers can now return pending transactions, flagged as such, when
their ids survive posting. They're imported unpaid right away and, once
the bank posts them, marked paid with their final amount and date.

- Settling happens once, so later edits stick, and one marked paid by
  hand is left alone. It goes through save() so a transfer's other side
  follows; no rules run again. reference_date moves only if it still
  followed the date.
- Each sync looks back to the oldest unpaid synced transaction, since a
  card purchase can post weeks later when the bill closes. Pending ones
  older than 60 days are taken as cancelled and no longer followed.
2026-11-08 16:09:23 -03:00

212 lines
7.9 KiB
Python

import logging
from datetime import timedelta
from django.db.models import Min
from django.utils import timezone
from apps.bank_sync.models import BankSyncAccount, BankSyncService
from apps.common.functions.decimals import truncate_decimal
from apps.common.middleware.thread_local import delete_current_user, write_current_user
from apps.rules.signals import transaction_created
from apps.transactions.models import Transaction
logger = logging.getLogger(__name__)
# Banks post late and back-date transactions, so every sync looks this far back
OVERLAP_DAYS = 7
# How far apart a synced transaction and the one it matches can be dated
MATCH_DAYS = 3
# Pending ones older than this are taken as cancelled and no longer followed. A
# card's bill cycle is at most ~40 days
PENDING_MAX_DAYS = 60
def refresh_remote_accounts(service: BankSyncService, provider=None) -> None:
"""Mirror the provider's account list, keeping existing links.
Accounts the provider no longer returns are dropped: their ids are gone for good.
"""
provider = provider or service.get_provider()
remote_accounts = {remote.id: remote for remote in provider.list_accounts()}
service.accounts.exclude(external_id__in=remote_accounts).delete()
for remote in remote_accounts.values():
link, _ = BankSyncAccount.objects.get_or_create(
service=service, external_id=remote.id
)
link.external_name = remote.name
link.remote_error = remote.error
link.available_currencies = sorted(
set(link.available_currencies) | set(remote.currencies)
)
link.save()
def _match_existing(link: BankSyncAccount, internal_id, type_, amount, date) -> bool:
"""Tag a transaction the user already added instead of creating a new one."""
candidates = Transaction.userless_all_objects.filter(
account=link.account,
type=type_,
amount=truncate_decimal(amount, link.account.currency.decimal_places),
date__range=(date - timedelta(days=MATCH_DAYS), date + timedelta(days=MATCH_DAYS)),
internal_id__isnull=True,
deleted=False,
)
best = min(candidates, key=lambda t: abs((t.date - date).days), default=None)
if best is None:
return False
# Not a user edit, so no transaction_updated (it would run "on update" rules)
Transaction.userless_all_objects.filter(pk=best.pk).update(
emit_signal=False, internal_id=internal_id
)
return True
def _settle(transaction: Transaction, amount, date) -> None:
"""The bank posted a transaction that was still unpaid: take its final values.
Happens once, so later edits stick. One marked paid by hand is left alone.
"""
if transaction.is_paid or transaction.deleted:
return
# Only when it still follows the date, so one picked by the user is kept
if transaction.reference_date == transaction.date.replace(day=1):
transaction.reference_date = date.replace(day=1)
transaction.is_paid = True
transaction.amount = amount
transaction.date = date
# save() keeps a transfer's other side in step. It sends no signal, and as a
# bank update rather than a user edit, rules don't run again
transaction.save()
def _sync_account(service: BankSyncService, link: BankSyncAccount, provider) -> int:
prefix = f"{service.service_type}:{link.external_id}:"
start = link.start_date
if link.last_synced_date:
start = max(start, link.last_synced_date - timedelta(days=OVERLAP_DAYS))
# Pending ones can post weeks later (a card's bill closing): look back far
# enough to see it happen
oldest_unpaid = Transaction.userless_all_objects.filter(
internal_id__startswith=prefix,
is_paid=False,
deleted=False,
date__gte=timezone.localdate() - timedelta(days=PENDING_MAX_DAYS),
).aggregate(Min("date"))["date__min"]
if oldest_unpaid:
start = min(start, oldest_unpaid)
created = 0
seen_currencies = set()
for remote in provider.get_transactions(link.external_id, start):
if remote.date < link.start_date:
continue
if remote.currency:
seen_currencies.add(remote.currency)
if link.currencies and remote.currency not in link.currencies:
continue
internal_id = prefix + remote.id
type_ = (
Transaction.Type.INCOME if remote.amount > 0 else Transaction.Type.EXPENSE
)
amount = abs(remote.amount)
# userless_all_objects: also sees soft deleted ones, so deleting a synced
# transaction doesn't bring it back on the next run
existing = Transaction.userless_all_objects.filter(
internal_id=internal_id
).first()
if existing:
if not remote.pending:
_settle(existing, amount, remote.date)
continue
if service.match_existing and _match_existing(
link, internal_id, type_, amount, remote.date
):
continue
transaction = Transaction.objects.create(
account=link.account,
owner=service.owner,
type=type_,
amount=amount,
date=remote.date,
description=remote.description[:500],
notes=remote.notes,
is_paid=not remote.pending,
internal_id=internal_id,
)
created += 1
if service.trigger_rules:
transaction_created.send(sender=transaction)
link.last_synced_date = timezone.localdate()
link.available_currencies = sorted(set(link.available_currencies) | seen_currencies)
link.save(update_fields=["last_synced_date", "available_currencies"])
return created
def sync_service(service: BankSyncService) -> int:
"""Import new transactions for every linked account. Returns how many.
A failing account doesn't stop the others. Any failure keeps last_sync as is,
so the service is retried on the next run.
"""
created = 0
errors = []
# Rules and the owned-object managers read the user from the thread
write_current_user(service.owner)
try:
provider = service.get_provider()
refresh_remote_accounts(service, provider)
links = service.accounts.filter(
account__isnull=False, start_date__isnull=False
).select_related("account__currency")
for link in links:
error = link.remote_error
if not error:
try:
created += _sync_account(service, link, provider)
except Exception as e:
logger.exception(f"Bank sync failed for {service.name} / {link}")
error = str(e)
link.last_error = error[:1000]
link.save(update_fields=["last_error"])
if error:
errors.append(f"{link}: {error}")
except Exception as e:
logger.exception(f"Bank sync failed for {service.name}")
errors.append(str(e))
finally:
delete_current_user()
if errors:
service.failure_count += 1
service.last_error = "\n".join(errors)[:1000]
service.save(update_fields=["failure_count", "last_error"])
else:
service.last_sync = timezone.now()
service.failure_count = 0
service.last_error = ""
service.save(update_fields=["last_sync", "failure_count", "last_error"])
logger.info(f"Bank sync for {service.name} created {created} transactions")
return created
def is_due(service: BankSyncService) -> bool:
if service.last_sync is None:
return True
# Hour granularity, the scheduler runs on the hour
elapsed = timezone.now().replace(minute=0, second=0, microsecond=0) - (
service.last_sync.replace(minute=0, second=0, microsecond=0)
)
return elapsed >= timedelta(hours=service.sync_interval)
def sync_due_services(force: bool = False) -> None:
for service in BankSyncService.objects.filter(is_active=True):
if force or is_due(service):
sync_service(service)