fix(import): make QIF deduplication account-aware and keep identical transactions

QIF imports deduplicate each transaction by storing a SHA-256 of its raw
lines as the transaction's internal_id and skipping rows whose hash is
already taken. The hash didn't include the account, which broke two
cases:

- Importing the same file into two accounts (possible since the account
  can be picked per run) skipped every row of the second import.
- Two real transactions with the same date, amount and payee in one file
  hashed to the same ID, so the second one was always dropped.

The internal ID now hashes the account's ID and the transaction's
occurrence (how many identical transactions came before it in the file)
together with the raw lines. Using the occurrence instead of the
position in the file keeps the IDs stable when a later export of the
same statement has more transactions before or between the repeated
ones.

Transactions imported before this change keep their old IDs. So that
re-importing an old file doesn't duplicate them, the first occurrence
of a transaction is also matched against the previous hash, limited to
the account being imported into. Repeats of identical transactions,
which the old hash always skipped, are imported now.

The duplicate check also looks at soft-deleted transactions, like the
CSV/Excel deduplication does. internal_id is unique, so a soft-deleted
transaction with the same ID used to fail the import with an integrity
error instead of being skipped.

Fixes #671
This commit is contained in:
Asish Kumar committed 2026-11-12 15:44:27 +05:30
1 parent 1c891a0a60
commit 70be7b3161
2 files changed
+173 -5

No files matched your search

+37 -5
View File
@@ -4,7 +4,9 @@ import logging
import os
import re
import zipfile
from collections import Counter
from django.db import transaction
from django.db.models import Q
from datetime import datetime, date
from decimal import Decimal, InvalidOperation
from pathlib import Path
@@ -968,12 +970,28 @@ class ImportService:
f"Invalid {self.settings.file_type.upper()} file format: {str(e)}"
)
@staticmethod
def _qif_internal_id(raw_content: str, account: Account, occurrence: int) -> str:
"""
Builds the internal ID of a QIF transaction from its raw lines.
The account is part of the hash so the same file can be imported into
different accounts, and the occurrence (how many identical transactions
came before it in the file) tells identical transactions apart, e.g. two
purchases with the same date, amount and payee.
"""
return hashlib.sha256(
f"{account.pk}:{occurrence}:{raw_content}".encode("utf-8")
).hexdigest()
def _parse_and_import_qif(self, content_lines: list[str], filename: str) -> None:
# Infer account from filename (remove extension)
account_name = os.path.splitext(os.path.basename(filename))[0]
current_transaction = {}
raw_lines_buffer = []
# How many times each transaction's raw lines have been seen in this file
occurrences = Counter()
account = (
self.import_run.account or Account.objects.filter(name=account_name).first()
@@ -994,17 +1012,31 @@ class ImportService:
if current_transaction:
# Deduplication using hash of raw lines
raw_content = "".join(raw_lines_buffer)
internal_id = hashlib.sha256(
raw_content.encode("utf-8")
).hexdigest()
occurrence = occurrences[raw_content]
occurrences[raw_content] += 1
internal_id = self._qif_internal_id(
raw_content, account, occurrence
)
# Reset buffer for next transaction
raw_lines_buffer = []
duplicate_filter = Q(internal_id=internal_id)
if occurrence == 0:
# Transactions imported before the account and the
# occurrence were part of the hash used the raw lines
# alone, and only ever kept the first of identical ones
duplicate_filter |= Q(
internal_id=hashlib.sha256(
raw_content.encode("utf-8")
).hexdigest(),
account=account,
)
try:
with transaction.atomic():
if Transaction.objects.filter(
internal_id=internal_id
if Transaction.all_objects.filter(
duplicate_filter
).exists():
self._increment_totals("skipped", 1)
self._log(
@@ -1,4 +1,6 @@
from datetime import date
from decimal import Decimal
import hashlib
import os
import shutil
from django.test import TestCase, override_settings
@@ -12,6 +14,30 @@ from apps.transactions.models import (
Transaction,
)
QIF_YAML_CONFIG = """
settings:
file_type: qif
importing: transactions
date_format: "%d/%m/%Y"
mapping: {}
"""
# A salary, then two purchases with the same date, amount and payee
IDENTICAL_TRANSACTIONS_CONTENT = """!Type:Bank
D04/01/2015
T100.00
PSalary
^
D05/01/2015
T-4.50
PCoffee Shop
^
D05/01/2015
T-4.50
PCoffee Shop
^
"""
class QIFImportTests(TestCase):
def setUp(self):
@@ -45,6 +71,23 @@ class QIFImportTests(TestCase):
if os.path.exists(self.test_dir):
shutil.rmtree(self.test_dir)
def _import_qif(self, content, account=None, filename="bradesco-checking.qif"):
file_path = os.path.join(self.test_dir, filename)
with open(file_path, "w", encoding="utf-8") as f:
f.write(content)
profile = ImportProfile.objects.create(
name="QIF Profile",
yaml_config=QIF_YAML_CONFIG,
version=ImportProfile.Versions.VERSION_1,
)
run = ImportRun.objects.create(
profile=profile, file_name=filename, account=account
)
ImportService(run).process_file(file_path)
run.refresh_from_db()
return run
def test_import_single_qif_valid_mapping(self):
content = """!Type:Bank
D04/01/2015
@@ -257,3 +300,96 @@ mapping: {}
self.assertEqual(
Transaction.objects.first().description, ""
) # empty desc if no memo
def test_import_same_file_into_two_accounts(self):
savings = Account.objects.create(
name="bradesco-savings",
group=self.group,
currency=self.currency,
owner=self.user,
)
into_checking = self._import_qif(
IDENTICAL_TRANSACTIONS_CONTENT, account=self.account
)
into_savings = self._import_qif(IDENTICAL_TRANSACTIONS_CONTENT, account=savings)
into_savings_again = self._import_qif(
IDENTICAL_TRANSACTIONS_CONTENT, account=savings
)
self.assertEqual(into_checking.successful_rows, 3)
self.assertEqual(into_savings.successful_rows, 3)
self.assertEqual(into_savings.skipped_rows, 0)
self.assertEqual(into_savings_again.successful_rows, 0)
self.assertEqual(into_savings_again.skipped_rows, 3)
self.assertEqual(Transaction.objects.filter(account=self.account).count(), 3)
self.assertEqual(Transaction.objects.filter(account=savings).count(), 3)
def test_import_identical_transactions_in_one_file(self):
first_run = self._import_qif(IDENTICAL_TRANSACTIONS_CONTENT)
self.assertEqual(first_run.successful_rows, 3)
self.assertEqual(first_run.skipped_rows, 0)
coffees = Transaction.objects.filter(entities__name="Coffee Shop")
self.assertEqual(coffees.count(), 2)
self.assertEqual(coffees.values("internal_id").distinct().count(), 2)
# Re-importing the file still recognizes both of them
second_run = self._import_qif(IDENTICAL_TRANSACTIONS_CONTENT)
self.assertEqual(second_run.successful_rows, 0)
self.assertEqual(second_run.skipped_rows, 3)
self.assertEqual(Transaction.objects.count(), 3)
def test_import_skips_transactions_imported_with_the_previous_hash(self):
# Imports made before the account was part of the hash used the raw
# lines alone, and skipped every repeat of identical transactions
legacy_internal_id = hashlib.sha256(
"D05/01/2015T-4.50PCoffee Shop^".encode("utf-8")
).hexdigest()
Transaction.objects.create(
account=self.account,
type=Transaction.Type.EXPENSE,
date=date(2015, 1, 5),
amount=Decimal("4.50"),
internal_id=legacy_internal_id,
)
savings = Account.objects.create(
name="bradesco-savings",
group=self.group,
currency=self.currency,
owner=self.user,
)
into_checking = self._import_qif(IDENTICAL_TRANSACTIONS_CONTENT)
into_savings = self._import_qif(IDENTICAL_TRANSACTIONS_CONTENT, account=savings)
# The first coffee is the one imported before, the second one was
# skipped back then and is imported now
self.assertEqual(into_checking.successful_rows, 2)
self.assertEqual(into_checking.skipped_rows, 1)
self.assertEqual(Transaction.objects.filter(account=self.account).count(), 3)
# The previous hash doesn't say which account it was for, so it only
# counts as a duplicate in the account the transaction is in
self.assertEqual(into_savings.successful_rows, 3)
self.assertEqual(into_savings.skipped_rows, 0)
self.assertEqual(Transaction.objects.filter(account=savings).count(), 3)
@override_settings(ENABLE_SOFT_DELETE=True)
def test_import_skips_soft_deleted_transactions(self):
content = """!Type:Bank
D04/01/2015
T100.00
POK
^
"""
self._import_qif(content)
Transaction.objects.get().delete()
self.assertTrue(Transaction.deleted_objects.exists())
run = self._import_qif(content)
self.assertEqual(run.status, ImportRun.Status.FINISHED)
self.assertEqual(run.successful_rows, 0)
self.assertEqual(run.skipped_rows, 1)
self.assertFalse(Transaction.objects.exists())