mirror of
https://github.com/eitchtee/WYGIWYH.git
synced 2026-11-16 22:01:28 +01:00
Merge pull request #672 from officialasishkumar/fix/qif-dedup-account
fix(import): make QIF deduplication account-aware and keep identical transactions
This commit is contained in:
2 files changed
+333
-7
No files matched your search
@@ -4,7 +4,9 @@ import logging
|
||||
import os
|
||||
import re
|
||||
import zipfile
|
||||
from collections import Counter
|
||||
from django.db import transaction
|
||||
from django.db.models import Q
|
||||
from datetime import datetime, date
|
||||
from decimal import Decimal, InvalidOperation
|
||||
from pathlib import Path
|
||||
@@ -39,6 +41,17 @@ from apps.import_app.schemas.v1 import (
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# QIF sections that hold transactions. Others, like category lists (!Type:Cat)
|
||||
# or memorized transactions (!Type:Memorized), are skipped
|
||||
QIF_TRANSACTION_SECTIONS = {
|
||||
"!type:bank",
|
||||
"!type:cash",
|
||||
"!type:ccard",
|
||||
"!type:invst",
|
||||
"!type:oth a",
|
||||
"!type:oth l",
|
||||
}
|
||||
|
||||
|
||||
class ImportService:
|
||||
def __init__(self, import_run: ImportRun):
|
||||
@@ -968,12 +981,32 @@ class ImportService:
|
||||
f"Invalid {self.settings.file_type.upper()} file format: {str(e)}"
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _qif_internal_id(raw_content: str, account: Account, occurrence: int) -> str:
|
||||
"""
|
||||
Builds the internal ID of a QIF transaction from its raw lines.
|
||||
|
||||
The account is part of the hash so the same file can be imported into
|
||||
different accounts, and the occurrence (how many identical transactions
|
||||
came before it in the file) tells identical transactions apart, e.g. two
|
||||
purchases with the same date, amount and payee.
|
||||
"""
|
||||
return hashlib.sha256(
|
||||
f"{account.pk}:{occurrence}:{raw_content}".encode("utf-8")
|
||||
).hexdigest()
|
||||
|
||||
def _parse_and_import_qif(self, content_lines: list[str], filename: str) -> None:
|
||||
# Infer account from filename (remove extension)
|
||||
account_name = os.path.splitext(os.path.basename(filename))[0]
|
||||
|
||||
current_transaction = {}
|
||||
raw_lines_buffer = []
|
||||
# How many times each transaction's raw lines have been seen in this file
|
||||
occurrences = Counter()
|
||||
legacy_contents_seen = set()
|
||||
in_account_list = False
|
||||
in_account_block = False
|
||||
in_other_section = False
|
||||
|
||||
account = (
|
||||
self.import_run.account or Account.objects.filter(name=account_name).first()
|
||||
@@ -988,23 +1021,66 @@ class ImportService:
|
||||
if not line:
|
||||
continue
|
||||
|
||||
# Account blocks (!Account, or an !Option:AutoSwitch account list)
|
||||
# and non-transaction sections use codes that mean something else
|
||||
# in a transaction, e.g. T is an account's type and D its description
|
||||
header = line.lower()
|
||||
if header == "!option:autoswitch":
|
||||
in_account_list = True
|
||||
elif header == "!clear:autoswitch":
|
||||
in_account_list = False
|
||||
elif header == "!account":
|
||||
in_account_block = True
|
||||
elif header.startswith("!type:"):
|
||||
in_other_section = header not in QIF_TRANSACTION_SECTIONS
|
||||
if in_other_section:
|
||||
self._log("info", f"Skipped {line} section in {filename}")
|
||||
if in_account_list or in_account_block or in_other_section:
|
||||
if line == "^":
|
||||
in_account_block = False
|
||||
raw_lines_buffer = []
|
||||
continue
|
||||
|
||||
raw_lines_buffer.append(line)
|
||||
|
||||
if line == "^":
|
||||
if current_transaction:
|
||||
# Deduplication using hash of raw lines
|
||||
raw_content = "".join(raw_lines_buffer)
|
||||
internal_id = hashlib.sha256(
|
||||
raw_content.encode("utf-8")
|
||||
).hexdigest()
|
||||
# Deduplication using hash of raw lines. Header lines (!Type,
|
||||
# !Account) are left out so a transaction hashes the same
|
||||
# whether or not it comes right after one
|
||||
legacy_content = "".join(raw_lines_buffer)
|
||||
raw_content = "".join(
|
||||
raw_line
|
||||
for raw_line in raw_lines_buffer
|
||||
if not raw_line.startswith("!")
|
||||
)
|
||||
occurrence = occurrences[raw_content]
|
||||
occurrences[raw_content] += 1
|
||||
internal_id = self._qif_internal_id(
|
||||
raw_content, account, occurrence
|
||||
)
|
||||
|
||||
# Reset buffer for next transaction
|
||||
raw_lines_buffer = []
|
||||
|
||||
duplicate_filter = Q(internal_id=internal_id)
|
||||
if legacy_content not in legacy_contents_seen:
|
||||
# Transactions imported before the account and the
|
||||
# occurrence were part of the hash used the raw lines
|
||||
# alone, headers included, and only ever kept the first
|
||||
# of identical ones
|
||||
legacy_contents_seen.add(legacy_content)
|
||||
duplicate_filter |= Q(
|
||||
internal_id=hashlib.sha256(
|
||||
legacy_content.encode("utf-8")
|
||||
).hexdigest(),
|
||||
account=account,
|
||||
)
|
||||
|
||||
try:
|
||||
with transaction.atomic():
|
||||
if Transaction.objects.filter(
|
||||
internal_id=internal_id
|
||||
if Transaction.all_objects.filter(
|
||||
duplicate_filter
|
||||
).exists():
|
||||
self._increment_totals("skipped", 1)
|
||||
self._log(
|
||||
|
||||
@@ -1,4 +1,6 @@
|
||||
from datetime import date
|
||||
from decimal import Decimal
|
||||
import hashlib
|
||||
import os
|
||||
import shutil
|
||||
from django.test import TestCase, override_settings
|
||||
@@ -12,6 +14,30 @@ from apps.transactions.models import (
|
||||
Transaction,
|
||||
)
|
||||
|
||||
QIF_YAML_CONFIG = """
|
||||
settings:
|
||||
file_type: qif
|
||||
importing: transactions
|
||||
date_format: "%d/%m/%Y"
|
||||
mapping: {}
|
||||
"""
|
||||
|
||||
# A salary, then two purchases with the same date, amount and payee
|
||||
IDENTICAL_TRANSACTIONS_CONTENT = """!Type:Bank
|
||||
D04/01/2015
|
||||
T100.00
|
||||
PSalary
|
||||
^
|
||||
D05/01/2015
|
||||
T-4.50
|
||||
PCoffee Shop
|
||||
^
|
||||
D05/01/2015
|
||||
T-4.50
|
||||
PCoffee Shop
|
||||
^
|
||||
"""
|
||||
|
||||
|
||||
class QIFImportTests(TestCase):
|
||||
def setUp(self):
|
||||
@@ -45,6 +71,23 @@ class QIFImportTests(TestCase):
|
||||
if os.path.exists(self.test_dir):
|
||||
shutil.rmtree(self.test_dir)
|
||||
|
||||
def _import_qif(self, content, account=None, filename="bradesco-checking.qif"):
|
||||
file_path = os.path.join(self.test_dir, filename)
|
||||
with open(file_path, "w", encoding="utf-8") as f:
|
||||
f.write(content)
|
||||
|
||||
profile = ImportProfile.objects.create(
|
||||
name="QIF Profile",
|
||||
yaml_config=QIF_YAML_CONFIG,
|
||||
version=ImportProfile.Versions.VERSION_1,
|
||||
)
|
||||
run = ImportRun.objects.create(
|
||||
profile=profile, file_name=filename, account=account
|
||||
)
|
||||
ImportService(run).process_file(file_path)
|
||||
run.refresh_from_db()
|
||||
return run
|
||||
|
||||
def test_import_single_qif_valid_mapping(self):
|
||||
content = """!Type:Bank
|
||||
D04/01/2015
|
||||
@@ -257,3 +300,210 @@ mapping: {}
|
||||
self.assertEqual(
|
||||
Transaction.objects.first().description, ""
|
||||
) # empty desc if no memo
|
||||
|
||||
def test_import_same_file_into_two_accounts(self):
|
||||
savings = Account.objects.create(
|
||||
name="bradesco-savings",
|
||||
group=self.group,
|
||||
currency=self.currency,
|
||||
owner=self.user,
|
||||
)
|
||||
|
||||
into_checking = self._import_qif(
|
||||
IDENTICAL_TRANSACTIONS_CONTENT, account=self.account
|
||||
)
|
||||
into_savings = self._import_qif(IDENTICAL_TRANSACTIONS_CONTENT, account=savings)
|
||||
into_savings_again = self._import_qif(
|
||||
IDENTICAL_TRANSACTIONS_CONTENT, account=savings
|
||||
)
|
||||
|
||||
self.assertEqual(into_checking.successful_rows, 3)
|
||||
self.assertEqual(into_savings.successful_rows, 3)
|
||||
self.assertEqual(into_savings.skipped_rows, 0)
|
||||
self.assertEqual(into_savings_again.successful_rows, 0)
|
||||
self.assertEqual(into_savings_again.skipped_rows, 3)
|
||||
self.assertEqual(Transaction.objects.filter(account=self.account).count(), 3)
|
||||
self.assertEqual(Transaction.objects.filter(account=savings).count(), 3)
|
||||
|
||||
def test_import_identical_transactions_in_one_file(self):
|
||||
first_run = self._import_qif(IDENTICAL_TRANSACTIONS_CONTENT)
|
||||
|
||||
self.assertEqual(first_run.successful_rows, 3)
|
||||
self.assertEqual(first_run.skipped_rows, 0)
|
||||
coffees = Transaction.objects.filter(entities__name="Coffee Shop")
|
||||
self.assertEqual(coffees.count(), 2)
|
||||
self.assertEqual(coffees.values("internal_id").distinct().count(), 2)
|
||||
|
||||
# Re-importing the file still recognizes both of them
|
||||
second_run = self._import_qif(IDENTICAL_TRANSACTIONS_CONTENT)
|
||||
|
||||
self.assertEqual(second_run.successful_rows, 0)
|
||||
self.assertEqual(second_run.skipped_rows, 3)
|
||||
self.assertEqual(Transaction.objects.count(), 3)
|
||||
|
||||
def test_import_skips_transactions_imported_with_the_previous_hash(self):
|
||||
# Imports made before the account was part of the hash used the raw
|
||||
# lines alone, and skipped every repeat of identical transactions
|
||||
legacy_internal_id = hashlib.sha256(
|
||||
"D05/01/2015T-4.50PCoffee Shop^".encode("utf-8")
|
||||
).hexdigest()
|
||||
Transaction.objects.create(
|
||||
account=self.account,
|
||||
type=Transaction.Type.EXPENSE,
|
||||
date=date(2015, 1, 5),
|
||||
amount=Decimal("4.50"),
|
||||
internal_id=legacy_internal_id,
|
||||
)
|
||||
savings = Account.objects.create(
|
||||
name="bradesco-savings",
|
||||
group=self.group,
|
||||
currency=self.currency,
|
||||
owner=self.user,
|
||||
)
|
||||
|
||||
into_checking = self._import_qif(IDENTICAL_TRANSACTIONS_CONTENT)
|
||||
into_savings = self._import_qif(IDENTICAL_TRANSACTIONS_CONTENT, account=savings)
|
||||
|
||||
# The first coffee is the one imported before, the second one was
|
||||
# skipped back then and is imported now
|
||||
self.assertEqual(into_checking.successful_rows, 2)
|
||||
self.assertEqual(into_checking.skipped_rows, 1)
|
||||
self.assertEqual(Transaction.objects.filter(account=self.account).count(), 3)
|
||||
# The previous hash doesn't say which account it was for, so it only
|
||||
# counts as a duplicate in the account the transaction is in
|
||||
self.assertEqual(into_savings.successful_rows, 3)
|
||||
self.assertEqual(into_savings.skipped_rows, 0)
|
||||
self.assertEqual(Transaction.objects.filter(account=savings).count(), 3)
|
||||
|
||||
def test_import_overlapping_exports_with_a_different_first_transaction(self):
|
||||
# The first coffee opens the second export, right after the header
|
||||
later_export = """!Type:Bank
|
||||
D05/01/2015
|
||||
T-4.50
|
||||
PCoffee Shop
|
||||
^
|
||||
D06/01/2015
|
||||
T-20.00
|
||||
PMarket
|
||||
^
|
||||
"""
|
||||
self._import_qif(IDENTICAL_TRANSACTIONS_CONTENT)
|
||||
|
||||
run = self._import_qif(later_export)
|
||||
|
||||
self.assertEqual(run.successful_rows, 1)
|
||||
self.assertEqual(run.skipped_rows, 1)
|
||||
self.assertEqual(Transaction.objects.count(), 4)
|
||||
|
||||
def test_import_skips_previous_hash_of_a_first_transaction_repeated_later(self):
|
||||
# The previous hash included the header in the first transaction, so a
|
||||
# first transaction and an identical later one were both imported
|
||||
content = """!Type:Bank
|
||||
D05/01/2015
|
||||
T-4.50
|
||||
PCoffee Shop
|
||||
^
|
||||
D05/01/2015
|
||||
T-4.50
|
||||
PCoffee Shop
|
||||
^
|
||||
"""
|
||||
for raw_content in (
|
||||
"!Type:BankD05/01/2015T-4.50PCoffee Shop^",
|
||||
"D05/01/2015T-4.50PCoffee Shop^",
|
||||
):
|
||||
Transaction.objects.create(
|
||||
account=self.account,
|
||||
type=Transaction.Type.EXPENSE,
|
||||
date=date(2015, 1, 5),
|
||||
amount=Decimal("4.50"),
|
||||
internal_id=hashlib.sha256(raw_content.encode("utf-8")).hexdigest(),
|
||||
)
|
||||
|
||||
run = self._import_qif(content)
|
||||
|
||||
self.assertEqual(run.successful_rows, 0)
|
||||
self.assertEqual(run.skipped_rows, 2)
|
||||
self.assertEqual(Transaction.objects.count(), 2)
|
||||
|
||||
def test_import_skips_account_blocks(self):
|
||||
# Quicken-style export: an account list, then an account block before
|
||||
# the transactions. T and D mean the account type and description there
|
||||
content = """!Option:AutoSwitch
|
||||
!Account
|
||||
NChecking
|
||||
TBank
|
||||
DMain checking account
|
||||
^
|
||||
NSavings
|
||||
TBank
|
||||
^
|
||||
!Clear:AutoSwitch
|
||||
!Account
|
||||
NChecking
|
||||
TBank
|
||||
DMain checking account
|
||||
^
|
||||
!Type:Bank
|
||||
D04/01/2015
|
||||
T100.00
|
||||
PSalary
|
||||
^
|
||||
"""
|
||||
run = self._import_qif(content)
|
||||
|
||||
self.assertEqual(run.status, ImportRun.Status.FINISHED)
|
||||
self.assertEqual(run.successful_rows, 1)
|
||||
self.assertEqual(run.failed_rows, 0)
|
||||
transaction = Transaction.objects.get()
|
||||
self.assertEqual(transaction.amount, Decimal("100.00"))
|
||||
self.assertEqual(transaction.date, date(2015, 1, 4))
|
||||
|
||||
def test_import_skips_non_transaction_sections(self):
|
||||
# Category and class lists, and memorized transactions (templates, not
|
||||
# real transactions), around a section of real ones
|
||||
content = """!Type:Cat
|
||||
NFood
|
||||
DGroceries and restaurants
|
||||
E
|
||||
^
|
||||
!Type:Class
|
||||
NPersonal
|
||||
DPersonal expenses
|
||||
^
|
||||
!Type:Bank
|
||||
D04/01/2015
|
||||
T100.00
|
||||
PSalary
|
||||
^
|
||||
!Type:Memorized
|
||||
KC
|
||||
T-50.00
|
||||
PRent
|
||||
^
|
||||
"""
|
||||
run = self._import_qif(content)
|
||||
|
||||
self.assertEqual(run.status, ImportRun.Status.FINISHED)
|
||||
self.assertEqual(run.successful_rows, 1)
|
||||
self.assertEqual(run.failed_rows, 0)
|
||||
self.assertEqual(Transaction.objects.get().amount, Decimal("100.00"))
|
||||
|
||||
@override_settings(ENABLE_SOFT_DELETE=True)
|
||||
def test_import_skips_soft_deleted_transactions(self):
|
||||
content = """!Type:Bank
|
||||
D04/01/2015
|
||||
T100.00
|
||||
POK
|
||||
^
|
||||
"""
|
||||
self._import_qif(content)
|
||||
Transaction.objects.get().delete()
|
||||
self.assertTrue(Transaction.deleted_objects.exists())
|
||||
|
||||
run = self._import_qif(content)
|
||||
|
||||
self.assertEqual(run.status, ImportRun.Status.FINISHED)
|
||||
self.assertEqual(run.successful_rows, 0)
|
||||
self.assertEqual(run.skipped_rows, 1)
|
||||
self.assertFalse(Transaction.objects.exists())
|
||||
Reference in new issue
Block a user