mirror of
https://github.com/eitchtee/WYGIWYH.git
synced 2026-11-20 07:31:44 +01:00
Provider registry modeled after automatic exchange rates: each provider declares its own settings fields, rendered dynamically in the service form. Lunch Flow is the first provider. - Map provider accounts to WYGIWYH accounts with a start date and an optional currency filter - Hourly task syncs due services; manual sync from the list - Deduplicates through Transaction.internal_id, optionally matching transactions already entered by hand - Per account errors and provider reported connection problems Refs #498
173 lines
6.4 KiB
Python
173 lines
6.4 KiB
Python
import logging
|
|
from datetime import timedelta
|
|
|
|
from django.utils import timezone
|
|
|
|
from apps.bank_sync.models import BankSyncAccount, BankSyncService
|
|
from apps.common.functions.decimals import truncate_decimal
|
|
from apps.common.middleware.thread_local import delete_current_user, write_current_user
|
|
from apps.rules.signals import transaction_created
|
|
from apps.transactions.models import Transaction
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
# Banks post late and back-date transactions, so every sync looks this far back
|
|
OVERLAP_DAYS = 7
|
|
# How far apart a synced transaction and the one it matches can be dated
|
|
MATCH_DAYS = 3
|
|
|
|
|
|
def refresh_remote_accounts(service: BankSyncService, provider=None) -> None:
|
|
"""Mirror the provider's account list, keeping existing links.
|
|
|
|
Accounts the provider no longer returns are dropped: their ids are gone for good.
|
|
"""
|
|
provider = provider or service.get_provider()
|
|
remote_accounts = {remote.id: remote for remote in provider.list_accounts()}
|
|
service.accounts.exclude(external_id__in=remote_accounts).delete()
|
|
|
|
for remote in remote_accounts.values():
|
|
link, _ = BankSyncAccount.objects.get_or_create(
|
|
service=service, external_id=remote.id
|
|
)
|
|
link.external_name = remote.name
|
|
link.remote_error = remote.error
|
|
link.available_currencies = sorted(
|
|
set(link.available_currencies) | set(remote.currencies)
|
|
)
|
|
link.save()
|
|
|
|
|
|
def _match_existing(link: BankSyncAccount, internal_id, type_, amount, date) -> bool:
|
|
"""Tag a transaction the user already added instead of creating a new one."""
|
|
candidates = Transaction.userless_all_objects.filter(
|
|
account=link.account,
|
|
type=type_,
|
|
amount=truncate_decimal(amount, link.account.currency.decimal_places),
|
|
date__range=(date - timedelta(days=MATCH_DAYS), date + timedelta(days=MATCH_DAYS)),
|
|
internal_id__isnull=True,
|
|
deleted=False,
|
|
)
|
|
best = min(candidates, key=lambda t: abs((t.date - date).days), default=None)
|
|
if best is None:
|
|
return False
|
|
# Not a user edit, so no transaction_updated (it would run "on update" rules)
|
|
Transaction.userless_all_objects.filter(pk=best.pk).update(
|
|
emit_signal=False, internal_id=internal_id
|
|
)
|
|
return True
|
|
|
|
|
|
def _sync_account(service: BankSyncService, link: BankSyncAccount, provider) -> int:
|
|
start = link.start_date
|
|
if link.last_synced_date:
|
|
start = max(start, link.last_synced_date - timedelta(days=OVERLAP_DAYS))
|
|
|
|
created = 0
|
|
seen_currencies = set()
|
|
for remote in provider.get_transactions(link.external_id, start):
|
|
if remote.date < link.start_date:
|
|
continue
|
|
if remote.currency:
|
|
seen_currencies.add(remote.currency)
|
|
if link.currencies and remote.currency not in link.currencies:
|
|
continue
|
|
|
|
internal_id = f"{service.service_type}:{link.external_id}:{remote.id}"
|
|
# userless_all_objects: also sees soft deleted ones, so deleting a synced
|
|
# transaction doesn't bring it back on the next run
|
|
if Transaction.userless_all_objects.filter(internal_id=internal_id).exists():
|
|
continue
|
|
|
|
type_ = (
|
|
Transaction.Type.INCOME if remote.amount > 0 else Transaction.Type.EXPENSE
|
|
)
|
|
amount = abs(remote.amount)
|
|
if service.match_existing and _match_existing(
|
|
link, internal_id, type_, amount, remote.date
|
|
):
|
|
continue
|
|
|
|
transaction = Transaction.objects.create(
|
|
account=link.account,
|
|
owner=service.owner,
|
|
type=type_,
|
|
amount=amount,
|
|
date=remote.date,
|
|
description=remote.description[:500],
|
|
notes=remote.notes,
|
|
is_paid=True,
|
|
internal_id=internal_id,
|
|
)
|
|
created += 1
|
|
if service.trigger_rules:
|
|
transaction_created.send(sender=transaction)
|
|
|
|
link.last_synced_date = timezone.localdate()
|
|
link.available_currencies = sorted(set(link.available_currencies) | seen_currencies)
|
|
link.save(update_fields=["last_synced_date", "available_currencies"])
|
|
return created
|
|
|
|
|
|
def sync_service(service: BankSyncService) -> int:
|
|
"""Import new transactions for every linked account. Returns how many.
|
|
|
|
A failing account doesn't stop the others. Any failure keeps last_sync as is,
|
|
so the service is retried on the next run.
|
|
"""
|
|
created = 0
|
|
errors = []
|
|
# Rules and the owned-object managers read the user from the thread
|
|
write_current_user(service.owner)
|
|
try:
|
|
provider = service.get_provider()
|
|
refresh_remote_accounts(service, provider)
|
|
links = service.accounts.filter(
|
|
account__isnull=False, start_date__isnull=False
|
|
).select_related("account__currency")
|
|
for link in links:
|
|
error = link.remote_error
|
|
if not error:
|
|
try:
|
|
created += _sync_account(service, link, provider)
|
|
except Exception as e:
|
|
logger.exception(f"Bank sync failed for {service.name} / {link}")
|
|
error = str(e)
|
|
link.last_error = error[:1000]
|
|
link.save(update_fields=["last_error"])
|
|
if error:
|
|
errors.append(f"{link}: {error}")
|
|
except Exception as e:
|
|
logger.exception(f"Bank sync failed for {service.name}")
|
|
errors.append(str(e))
|
|
finally:
|
|
delete_current_user()
|
|
|
|
if errors:
|
|
service.failure_count += 1
|
|
service.last_error = "\n".join(errors)[:1000]
|
|
service.save(update_fields=["failure_count", "last_error"])
|
|
else:
|
|
service.last_sync = timezone.now()
|
|
service.failure_count = 0
|
|
service.last_error = ""
|
|
service.save(update_fields=["last_sync", "failure_count", "last_error"])
|
|
logger.info(f"Bank sync for {service.name} created {created} transactions")
|
|
return created
|
|
|
|
|
|
def is_due(service: BankSyncService) -> bool:
|
|
if service.last_sync is None:
|
|
return True
|
|
# Hour granularity, the scheduler runs on the hour
|
|
elapsed = timezone.now().replace(minute=0, second=0, microsecond=0) - (
|
|
service.last_sync.replace(minute=0, second=0, microsecond=0)
|
|
)
|
|
return elapsed >= timedelta(hours=service.sync_interval)
|
|
|
|
|
|
def sync_due_services(force: bool = False) -> None:
|
|
for service in BankSyncService.objects.filter(is_active=True):
|
|
if force or is_due(service):
|
|
sync_service(service)
|