Merge pull request #672 from officialasishkumar/fix/qif-dedup-account

fix(import): make QIF deduplication account-aware and keep identical transactions
This commit is contained in:
Herculino Trotta authored and GitHub committed 2026-11-14 12:20:49 -03:00
commit eb51e240ac
2 files changed
+333 -7

No files matched your search

+83 -7
View File
@@ -4,7 +4,9 @@ import logging
import os
import re
import zipfile
from collections import Counter
from django.db import transaction
from django.db.models import Q
from datetime import datetime, date
from decimal import Decimal, InvalidOperation
from pathlib import Path
@@ -39,6 +41,17 @@ from apps.import_app.schemas.v1 import (
logger = logging.getLogger(__name__)
# QIF sections that hold transactions. Others, like category lists (!Type:Cat)
# or memorized transactions (!Type:Memorized), are skipped
QIF_TRANSACTION_SECTIONS = {
"!type:bank",
"!type:cash",
"!type:ccard",
"!type:invst",
"!type:oth a",
"!type:oth l",
}
class ImportService:
def __init__(self, import_run: ImportRun):
@@ -968,12 +981,32 @@ class ImportService:
f"Invalid {self.settings.file_type.upper()} file format: {str(e)}"
)
@staticmethod
def _qif_internal_id(raw_content: str, account: Account, occurrence: int) -> str:
"""
Builds the internal ID of a QIF transaction from its raw lines.
The account is part of the hash so the same file can be imported into
different accounts, and the occurrence (how many identical transactions
came before it in the file) tells identical transactions apart, e.g. two
purchases with the same date, amount and payee.
"""
return hashlib.sha256(
f"{account.pk}:{occurrence}:{raw_content}".encode("utf-8")
).hexdigest()
def _parse_and_import_qif(self, content_lines: list[str], filename: str) -> None:
# Infer account from filename (remove extension)
account_name = os.path.splitext(os.path.basename(filename))[0]
current_transaction = {}
raw_lines_buffer = []
# How many times each transaction's raw lines have been seen in this file
occurrences = Counter()
legacy_contents_seen = set()
in_account_list = False
in_account_block = False
in_other_section = False
account = (
self.import_run.account or Account.objects.filter(name=account_name).first()
@@ -988,23 +1021,66 @@ class ImportService:
if not line:
continue
# Account blocks (!Account, or an !Option:AutoSwitch account list)
# and non-transaction sections use codes that mean something else
# in a transaction, e.g. T is an account's type and D its description
header = line.lower()
if header == "!option:autoswitch":
in_account_list = True
elif header == "!clear:autoswitch":
in_account_list = False
elif header == "!account":
in_account_block = True
elif header.startswith("!type:"):
in_other_section = header not in QIF_TRANSACTION_SECTIONS
if in_other_section:
self._log("info", f"Skipped {line} section in {filename}")
if in_account_list or in_account_block or in_other_section:
if line == "^":
in_account_block = False
raw_lines_buffer = []
continue
raw_lines_buffer.append(line)
if line == "^":
if current_transaction:
# Deduplication using hash of raw lines
raw_content = "".join(raw_lines_buffer)
internal_id = hashlib.sha256(
raw_content.encode("utf-8")
).hexdigest()
# Deduplication using hash of raw lines. Header lines (!Type,
# !Account) are left out so a transaction hashes the same
# whether or not it comes right after one
legacy_content = "".join(raw_lines_buffer)
raw_content = "".join(
raw_line
for raw_line in raw_lines_buffer
if not raw_line.startswith("!")
)
occurrence = occurrences[raw_content]
occurrences[raw_content] += 1
internal_id = self._qif_internal_id(
raw_content, account, occurrence
)
# Reset buffer for next transaction
raw_lines_buffer = []
duplicate_filter = Q(internal_id=internal_id)
if legacy_content not in legacy_contents_seen:
# Transactions imported before the account and the
# occurrence were part of the hash used the raw lines
# alone, headers included, and only ever kept the first
# of identical ones
legacy_contents_seen.add(legacy_content)
duplicate_filter |= Q(
internal_id=hashlib.sha256(
legacy_content.encode("utf-8")
).hexdigest(),
account=account,
)
try:
with transaction.atomic():
if Transaction.objects.filter(
internal_id=internal_id
if Transaction.all_objects.filter(
duplicate_filter
).exists():
self._increment_totals("skipped", 1)
self._log(
@@ -1,4 +1,6 @@
from datetime import date
from decimal import Decimal
import hashlib
import os
import shutil
from django.test import TestCase, override_settings
@@ -12,6 +14,30 @@ from apps.transactions.models import (
Transaction,
)
QIF_YAML_CONFIG = """
settings:
file_type: qif
importing: transactions
date_format: "%d/%m/%Y"
mapping: {}
"""
# A salary, then two purchases with the same date, amount and payee
IDENTICAL_TRANSACTIONS_CONTENT = """!Type:Bank
D04/01/2015
T100.00
PSalary
^
D05/01/2015
T-4.50
PCoffee Shop
^
D05/01/2015
T-4.50
PCoffee Shop
^
"""
class QIFImportTests(TestCase):
def setUp(self):
@@ -45,6 +71,23 @@ class QIFImportTests(TestCase):
if os.path.exists(self.test_dir):
shutil.rmtree(self.test_dir)
def _import_qif(self, content, account=None, filename="bradesco-checking.qif"):
file_path = os.path.join(self.test_dir, filename)
with open(file_path, "w", encoding="utf-8") as f:
f.write(content)
profile = ImportProfile.objects.create(
name="QIF Profile",
yaml_config=QIF_YAML_CONFIG,
version=ImportProfile.Versions.VERSION_1,
)
run = ImportRun.objects.create(
profile=profile, file_name=filename, account=account
)
ImportService(run).process_file(file_path)
run.refresh_from_db()
return run
def test_import_single_qif_valid_mapping(self):
content = """!Type:Bank
D04/01/2015
@@ -257,3 +300,210 @@ mapping: {}
self.assertEqual(
Transaction.objects.first().description, ""
) # empty desc if no memo
def test_import_same_file_into_two_accounts(self):
savings = Account.objects.create(
name="bradesco-savings",
group=self.group,
currency=self.currency,
owner=self.user,
)
into_checking = self._import_qif(
IDENTICAL_TRANSACTIONS_CONTENT, account=self.account
)
into_savings = self._import_qif(IDENTICAL_TRANSACTIONS_CONTENT, account=savings)
into_savings_again = self._import_qif(
IDENTICAL_TRANSACTIONS_CONTENT, account=savings
)
self.assertEqual(into_checking.successful_rows, 3)
self.assertEqual(into_savings.successful_rows, 3)
self.assertEqual(into_savings.skipped_rows, 0)
self.assertEqual(into_savings_again.successful_rows, 0)
self.assertEqual(into_savings_again.skipped_rows, 3)
self.assertEqual(Transaction.objects.filter(account=self.account).count(), 3)
self.assertEqual(Transaction.objects.filter(account=savings).count(), 3)
def test_import_identical_transactions_in_one_file(self):
first_run = self._import_qif(IDENTICAL_TRANSACTIONS_CONTENT)
self.assertEqual(first_run.successful_rows, 3)
self.assertEqual(first_run.skipped_rows, 0)
coffees = Transaction.objects.filter(entities__name="Coffee Shop")
self.assertEqual(coffees.count(), 2)
self.assertEqual(coffees.values("internal_id").distinct().count(), 2)
# Re-importing the file still recognizes both of them
second_run = self._import_qif(IDENTICAL_TRANSACTIONS_CONTENT)
self.assertEqual(second_run.successful_rows, 0)
self.assertEqual(second_run.skipped_rows, 3)
self.assertEqual(Transaction.objects.count(), 3)
def test_import_skips_transactions_imported_with_the_previous_hash(self):
# Imports made before the account was part of the hash used the raw
# lines alone, and skipped every repeat of identical transactions
legacy_internal_id = hashlib.sha256(
"D05/01/2015T-4.50PCoffee Shop^".encode("utf-8")
).hexdigest()
Transaction.objects.create(
account=self.account,
type=Transaction.Type.EXPENSE,
date=date(2015, 1, 5),
amount=Decimal("4.50"),
internal_id=legacy_internal_id,
)
savings = Account.objects.create(
name="bradesco-savings",
group=self.group,
currency=self.currency,
owner=self.user,
)
into_checking = self._import_qif(IDENTICAL_TRANSACTIONS_CONTENT)
into_savings = self._import_qif(IDENTICAL_TRANSACTIONS_CONTENT, account=savings)
# The first coffee is the one imported before, the second one was
# skipped back then and is imported now
self.assertEqual(into_checking.successful_rows, 2)
self.assertEqual(into_checking.skipped_rows, 1)
self.assertEqual(Transaction.objects.filter(account=self.account).count(), 3)
# The previous hash doesn't say which account it was for, so it only
# counts as a duplicate in the account the transaction is in
self.assertEqual(into_savings.successful_rows, 3)
self.assertEqual(into_savings.skipped_rows, 0)
self.assertEqual(Transaction.objects.filter(account=savings).count(), 3)
def test_import_overlapping_exports_with_a_different_first_transaction(self):
# The first coffee opens the second export, right after the header
later_export = """!Type:Bank
D05/01/2015
T-4.50
PCoffee Shop
^
D06/01/2015
T-20.00
PMarket
^
"""
self._import_qif(IDENTICAL_TRANSACTIONS_CONTENT)
run = self._import_qif(later_export)
self.assertEqual(run.successful_rows, 1)
self.assertEqual(run.skipped_rows, 1)
self.assertEqual(Transaction.objects.count(), 4)
def test_import_skips_previous_hash_of_a_first_transaction_repeated_later(self):
# The previous hash included the header in the first transaction, so a
# first transaction and an identical later one were both imported
content = """!Type:Bank
D05/01/2015
T-4.50
PCoffee Shop
^
D05/01/2015
T-4.50
PCoffee Shop
^
"""
for raw_content in (
"!Type:BankD05/01/2015T-4.50PCoffee Shop^",
"D05/01/2015T-4.50PCoffee Shop^",
):
Transaction.objects.create(
account=self.account,
type=Transaction.Type.EXPENSE,
date=date(2015, 1, 5),
amount=Decimal("4.50"),
internal_id=hashlib.sha256(raw_content.encode("utf-8")).hexdigest(),
)
run = self._import_qif(content)
self.assertEqual(run.successful_rows, 0)
self.assertEqual(run.skipped_rows, 2)
self.assertEqual(Transaction.objects.count(), 2)
def test_import_skips_account_blocks(self):
# Quicken-style export: an account list, then an account block before
# the transactions. T and D mean the account type and description there
content = """!Option:AutoSwitch
!Account
NChecking
TBank
DMain checking account
^
NSavings
TBank
^
!Clear:AutoSwitch
!Account
NChecking
TBank
DMain checking account
^
!Type:Bank
D04/01/2015
T100.00
PSalary
^
"""
run = self._import_qif(content)
self.assertEqual(run.status, ImportRun.Status.FINISHED)
self.assertEqual(run.successful_rows, 1)
self.assertEqual(run.failed_rows, 0)
transaction = Transaction.objects.get()
self.assertEqual(transaction.amount, Decimal("100.00"))
self.assertEqual(transaction.date, date(2015, 1, 4))
def test_import_skips_non_transaction_sections(self):
# Category and class lists, and memorized transactions (templates, not
# real transactions), around a section of real ones
content = """!Type:Cat
NFood
DGroceries and restaurants
E
^
!Type:Class
NPersonal
DPersonal expenses
^
!Type:Bank
D04/01/2015
T100.00
PSalary
^
!Type:Memorized
KC
T-50.00
PRent
^
"""
run = self._import_qif(content)
self.assertEqual(run.status, ImportRun.Status.FINISHED)
self.assertEqual(run.successful_rows, 1)
self.assertEqual(run.failed_rows, 0)
self.assertEqual(Transaction.objects.get().amount, Decimal("100.00"))
@override_settings(ENABLE_SOFT_DELETE=True)
def test_import_skips_soft_deleted_transactions(self):
content = """!Type:Bank
D04/01/2015
T100.00
POK
^
"""
self._import_qif(content)
Transaction.objects.get().delete()
self.assertTrue(Transaction.deleted_objects.exists())
run = self._import_qif(content)
self.assertEqual(run.status, ImportRun.Status.FINISHED)
self.assertEqual(run.successful_rows, 0)
self.assertEqual(run.skipped_rows, 1)
self.assertFalse(Transaction.objects.exists())